@mnemonik/shared 7.119.2 → 7.120.0

package/src/codeScanner.ts

@@ -18,6 +18,18 @@ import {
   type AstChunk,
 } from './ast/astChunker.js';
 import { astArtifactReport, resolveAstLanguage } from './ast/grammars.js';
+import {
+  classifyForIndex,
+  gitattributesCanReinclude,
+  parseGitattributes,
+  softDefaultForContent,
+  softDefaultForDirectory,
+  type GitattributesLayer,
+  type GitattributesRule,
+  type IgnoreFileVerdict,
+  type IndexClassification,
+  type IndexExclusionReason,
+} from './indexClassifier.js';
 
 /**
  * File operation timeout (5 seconds) to prevent hanging on slow/unresponsive filesystems
@@ -106,6 +118,31 @@ export interface ScanOptions {
  */
 interface WalkState {
   complete: boolean;
+  /**
+   * When present, every rule file the walk reads is recorded here by
+   * directory, so the daemon can upload the lines it actually applied.
+   */
+  rules?: Map<string, IndexRuleFiles>;
+}
+
+/** The rule-file lines one directory contributed to a walk. */
+export interface IndexRuleFiles {
+  /** .gitignore + .mnemonikignore lines, unioned, in read order. */
+  ignore: string[];
+  /** .gitattributes lines, in read order. */
+  attributes: string[];
+}
+
+/**
+ * A file a walk declined to index because a soft default excluded it.
+ * `stage: 'content'` means the file was read and its content decided.
+ */
+export interface IndexExcludedFile {
+  /** Root-relative POSIX path, spelled as a chunk's `filePath`. */
+  path: string;
+  reason: IndexExclusionReason;
+  rule: string;
+  stage: 'path' | 'content';
 }
 
 /**
@@ -190,6 +227,9 @@ const BUILT_IN_IGNORE_FILE_PATTERNS: readonly string[] = [
   // `isSecretFile` below, not here. This glob engine cannot express negation
   // (`.env.example` must stay collectable) and its unanchored substring
   // matching would make `.env.*` also swallow `.environment.ts`.
+  //
+  // `/tests/fixtures/*` used to live here. It is now a soft default in
+  // indexClassifier.ts (test fixtures, R2), which a project can re-include.
   '.DS_Store',
   '*.log',
   '*.lock',
@@ -200,7 +240,6 @@ const BUILT_IN_IGNORE_FILE_PATTERNS: readonly string[] = [
   '*.bundle.js',
   '*.legacy.js',
   '*.map',
-  '/tests/fixtures/*',
 ];
 
 /**
@@ -271,6 +310,9 @@ export function isSecretFile(relPath: string): boolean {
  */
 const IGNORE_FILE_NAMES = ['.gitignore', '.mnemonikignore'] as const;
 
+/** Per-directory attributes file: linguist overrides and binary marks (indexClassifier.ts). */
+const ATTRIBUTES_FILE_NAME = '.gitattributes';
+
 /**
  * Every file type the scanner will chunk, by extension. Matched
  * case-INSENSITIVELY (see `isIncludedExtension`), so lowercase spellings here
@@ -587,12 +629,11 @@ export const AUTHORITY_FILE_MATCHERS: Array<(relPath: string) => boolean> = [
 ];
 
 /**
- * Segment-anchored match for the `tests/fixtures/` ignore pattern above -
- * matches at the path root or immediately after a `/`, mirroring exactly
- * what `shouldIgnore` compiles `'/tests/fixtures/*'` to. Exported so
- * server-side consumers (e.g. the `/scan/push` route) and query-time
- * consumers (e.g. `MemoryManager.searchCodePointers`'s hygiene exclusions)
- * share one definition instead of hand-rolling copies that can drift.
+ * Segment-anchored match for `tests/fixtures/` - matches at the path root or
+ * immediately after a `/`. Indexing no longer uses it (the fixture rule is the
+ * soft default in indexClassifier.ts, which a project can override); it stays
+ * for query-time hygiene over rows indexed before that rule existed
+ * (`MemoryManager.searchCodePointers`).
  */
 export const FIXTURE_PATH_RE = /(^|\/)tests\/fixtures\//;
 export function isFixturePath(path: string): boolean {
@@ -666,7 +707,12 @@ export async function isGitBoundary(dirPath: string): Promise<boolean> {
  */
 interface IgnoreLayer {
   baseRel: string;
-  ig: Ignore;
+  /** Compiled .gitignore + .mnemonikignore; null when this directory has neither. */
+  ig: Ignore | null;
+  /** Any `!pattern` line: a path under here may be re-included from a soft default. */
+  hasNegation: boolean;
+  /** Parsed .gitattributes; null when this directory has none. */
+  attributes: GitattributesRule[] | null;
 }
 type IgnoreStack = IgnoreLayer[];
 
@@ -789,12 +835,13 @@ export class CodeScanner {
   }
 
   /**
-   * Read `.gitignore` + `.mnemonikignore` in `absDir` and compile them into one
-   * ignore layer (the two files are unioned - `.mnemonikignore` is just an
-   * additional gitignore-syntax exclusion list). Returns null when neither file
-   * exists. FAIL-CLOSED: a non-ENOENT read error (EISDIR, EACCES, timeout)
-   * latches `walk.complete = false` so callers deriving *removals* from the walk
-   * treat it as untrustworthy rather than silently ignoring nothing.
+   * Read `.gitignore` + `.mnemonikignore` + `.gitattributes` in `absDir` and
+   * compile them into one layer (the two ignore files are unioned -
+   * `.mnemonikignore` is just an additional gitignore-syntax exclusion list).
+   * Returns null when none of the three exists. FAIL-CLOSED: a non-ENOENT read
+   * error (EISDIR, EACCES, timeout) latches `walk.complete = false` so callers
+   * deriving *removals* from the walk treat it as untrustworthy rather than
+   * silently ignoring nothing.
    */
   private async loadIgnoreLayer(
     absDir: string,
@@ -803,8 +850,7 @@ export class CodeScanner {
     rootPath: string
   ): Promise<IgnoreLayer | null> {
     if (this.isProtected(absDir)) return null;
-    const patterns: string[] = [];
-    for (const name of IGNORE_FILE_NAMES) {
+    const read = async (name: string): Promise<string[] | null> => {
       const filePath = join(absDir, name);
       try {
         const content = await withTimeout(
@@ -812,32 +858,50 @@ export class CodeScanner {
           FILE_OP_TIMEOUT_MS,
           `readFile timed out: ${filePath}`
         );
-        for (const line of content.split(/\r?\n/)) patterns.push(line);
+        return content.split(/\r?\n/);
       } catch (err) {
-        if ((err as Error).message === 'symlink_skipped') continue;
+        if ((err as Error).message === 'symlink_skipped') return null;
         if ((err as NodeJS.ErrnoException)?.code !== 'ENOENT') {
           walk.complete = false;
-          logDebug('Ignore-file read failed (fail-closed)', { path: filePath, err });
+          logDebug('Rule-file read failed (fail-closed)', { path: filePath, err });
         }
+        return null;
       }
+    };
+    const patterns: string[] = [];
+    for (const name of IGNORE_FILE_NAMES) {
+      const lines = await read(name);
+      if (lines) patterns.push(...lines);
+    }
+    const attributeLines = (await read(ATTRIBUTES_FILE_NAME)) ?? [];
+    if (patterns.length > 0 || attributeLines.length > 0) {
+      walk.rules?.set(baseRel, { ignore: patterns, attributes: attributeLines });
     }
+    const attributes =
+      attributeLines.length > 0 ? parseGitattributes(attributeLines.join('\n')) : [];
     // `ignore` tolerates blank/comment lines; null when nothing meaningful.
-    if (patterns.length === 0) return null;
-    const ig = ignore().add(patterns);
-    return { baseRel, ig };
+    if (patterns.length === 0 && attributes.length === 0) return null;
+    return {
+      baseRel,
+      ig: patterns.length > 0 ? ignore().add(patterns) : null,
+      hasNegation: patterns.some((line) => line.trimStart().startsWith('!')),
+      attributes: attributes.length > 0 ? attributes : null,
+    };
   }
 
   /**
-   * True when `entryRel` (relative to the scan root) is excluded by any ignore
-   * layer in `stack`. Tested deepest-first: the first layer that explicitly
-   * ignores OR re-includes (via a `!negation`) wins, matching git's rule that a
-   * deeper `.gitignore` overrides a shallower one. Directories are tested with a
-   * trailing slash so `foo/` dir-only rules match the directory itself.
+   * What the .gitignore / .mnemonikignore layers in `stack` say about
+   * `entryRel` (relative to the scan root). Tested deepest-first: the first
+   * layer that explicitly ignores OR re-includes (via a `!negation`) wins,
+   * matching git's rule that a deeper `.gitignore` overrides a shallower one.
+   * Directories are tested with a trailing slash so `foo/` dir-only rules
+   * match the directory itself. `'unignored'` means a negation matched this
+   * path itself; a re-included parent directory leaves its files `'unmatched'`.
    */
-  private ignoredByStack(entryRel: string, isDir: boolean, stack: IgnoreStack): boolean {
+  private ignoreVerdict(entryRel: string, isDir: boolean, stack: IgnoreStack): IgnoreFileVerdict {
     for (let i = stack.length - 1; i >= 0; i--) {
       const layer = stack[i];
-      if (!layer) continue;
+      if (!layer?.ig) continue;
       const { baseRel, ig } = layer;
       let sub: string;
       if (baseRel === '') {
@@ -849,10 +913,57 @@ export class CodeScanner {
       }
       if (!sub) continue;
       const res = ig.test(isDir ? sub + '/' : sub);
-      if (res.ignored) return true;
-      if (res.unignored) return false;
+      if (res.ignored) return 'ignored';
+      if (res.unignored) return 'unignored';
     }
-    return false;
+    return 'unmatched';
+  }
+
+  private ignoredByStack(entryRel: string, isDir: boolean, stack: IgnoreStack): boolean {
+    return this.ignoreVerdict(entryRel, isDir, stack) === 'ignored';
+  }
+
+  private static attributeLayers(stack: IgnoreStack): GitattributesLayer[] {
+    return stack.flatMap((layer) =>
+      layer.attributes ? [{ baseRel: layer.baseRel, rules: layer.attributes }] : []
+    );
+  }
+
+  /**
+   * Skip this directory? User-ignored directories are skipped as before. A
+   * directory a soft default excludes (vendor/, generated/, ...) is pruned too -
+   * nothing under it can be indexed - UNLESS a rule above it could re-include
+   * something inside it: a `!pattern` or a `-linguist-*` attribute. Then the
+   * walk descends and judges each file, so the override works. Rule files
+   * INSIDE a pruned directory are never read: a vendored library's own
+   * .gitignore negations are not the project's intent.
+   */
+  private skipDirectory(relPath: string, stack: IgnoreStack): boolean {
+    const posixRel = relPath.split(sep).join('/');
+    const verdict = this.ignoreVerdict(posixRel, true, stack);
+    if (verdict === 'ignored') return true;
+    if (verdict === 'unignored') return false;
+    if (!softDefaultForDirectory(posixRel)) return false;
+    const canReinclude =
+      stack.some((layer) => layer.hasNegation) ||
+      gitattributesCanReinclude(CodeScanner.attributeLayers(stack));
+    return !canReinclude;
+  }
+
+  /**
+   * The index decision for a file from its path and the rule stack: `null`
+   * when a user ignore rule excludes it, otherwise the classifier's answer
+   * (precedence in indexClassifier.ts). Content rules are the caller's next
+   * step for an included path without an `override`.
+   */
+  private classifyEntry(relPath: string, stack: IgnoreStack): IndexClassification | null {
+    const posixRel = relPath.split(sep).join('/');
+    const verdict = this.ignoreVerdict(posixRel, false, stack);
+    if (verdict === 'ignored') return null;
+    return classifyForIndex(posixRel, {
+      ignoreFileVerdict: verdict,
+      attributes: CodeScanner.attributeLayers(stack),
+    });
   }
 
   /**
@@ -889,6 +1000,31 @@ export class CodeScanner {
     return stack;
   }
 
+  /**
+   * A predicate for walkers outside this class (the daemon's file watcher):
+   * true when the walk would descend into `absDir`, by the same rules
+   * `listFiles` applies - user ignore files and soft-default directories,
+   * with their overrides. Rule files are read once per directory for the life
+   * of the returned predicate; take a fresh one after a rule file changes.
+   * Directories outside `rootPath`, and the root itself, always pass.
+   */
+  directoryFilter(rootPath: string): (absDir: string) => Promise<boolean> {
+    const cache = new Map<string, IgnoreLayer | null>();
+    return async (absDir: string): Promise<boolean> => {
+      try {
+        const root = await this.canonicalRoot(rootPath);
+        const rel = relative(root, absDir).split(sep).join('/');
+        if (!rel || rel.startsWith('..')) return true;
+        const stack = await this.buildIgnoreStackForFile(rel, root, cache, { complete: true });
+        return !this.skipDirectory(rel, stack);
+      } catch {
+        // Fail open: watching a directory the walk would skip costs a watch
+        // handle; not watching one it would index loses changes.
+        return true;
+      }
+    };
+  }
+
   /**
    * True when any directory strictly between `rootPath` and `filePath`
    * carries its own `.git` entry (see isGitBoundary). Paths that don't
@@ -1019,15 +1155,20 @@ export class CodeScanner {
    * redaction guarantee as `scanFiles`). `chunkless` lists the Markdown and
    * instruction files that chunked to nothing, as `scanFiles` reports them.
    */
-  async scanDirectoryWithStatus(
-    rootPath: string
-  ): Promise<{ chunks: CodeChunk[]; complete: boolean; chunkless: string[] }> {
+  async scanDirectoryWithStatus(rootPath: string): Promise<{
+    chunks: CodeChunk[];
+    complete: boolean;
+    chunkless: string[];
+    /** Files a soft default excluded by their content (path exclusions are not listed). */
+    excluded: IndexExcludedFile[];
+  }> {
     rootPath = await this.canonicalRoot(rootPath);
     const chunks: CodeChunk[] = [];
     const chunkless: string[] = [];
+    const excluded: IndexExcludedFile[] = [];
     const walk: WalkState = { complete: true };
-    await this.traverseDirectory(rootPath, rootPath, chunks, 0, walk, [], chunkless);
-    return { chunks: this.scrubChunks(chunks), complete: walk.complete, chunkless };
+    await this.traverseDirectory(rootPath, rootPath, chunks, 0, walk, [], chunkless, excluded);
+    return { chunks: this.scrubChunks(chunks), complete: walk.complete, chunkless, excluded };
   }
 
   /**
@@ -1058,24 +1199,50 @@ export class CodeScanner {
    * MUST NOT derive removedFiles from an incomplete inventory (a truncated
    * walk otherwise reads as mass deletion and deprecates live memories).
    */
-  async listFilesWithStatus(rootPath: string): Promise<{ paths: string[]; complete: boolean }> {
+  async listFilesWithStatus(rootPath: string): Promise<{
+    paths: string[];
+    complete: boolean;
+    /**
+     * Listed paths a project rule re-included (a `!pattern` or a
+     * `-linguist-*` attribute). Content rules do not apply to them; every
+     * other listed path still answers to the content rules when read.
+     */
+    overridden: Set<string>;
+  }> {
     rootPath = await this.canonicalRoot(rootPath);
     const paths: string[] = [];
-    const walk: WalkState = { complete: true };
-    await this.traversePaths(rootPath, rootPath, paths, 0, walk);
+    const overridden = new Set<string>();
+    const walk: WalkState = { complete: true, rules: new Map() };
+    await this.traversePaths(rootPath, rootPath, paths, 0, walk, [], overridden);
+    // Only a complete walk replaces the record: a partial one may be missing
+    // rule files, and uploading it would tell the server fewer rules exist.
+    if (walk.complete && walk.rules) this.ruleFilesByRoot.set(rootPath, walk.rules);
     return {
       paths: paths.filter((p) => !/(^|[/\\])\.\.([/\\]|$)/.test(p)),
       complete: walk.complete,
+      overridden,
     };
   }
 
+  /**
+   * The rule files the last complete `listFilesWithStatus` walk of `rootPath`
+   * applied, by directory ('' = root). Null before the first complete walk.
+   * The daemon uploads these lines so the server judges paths as it did.
+   */
+  async indexRuleFiles(rootPath: string): Promise<ReadonlyMap<string, IndexRuleFiles> | null> {
+    return this.ruleFilesByRoot.get(await this.canonicalRoot(rootPath)) ?? null;
+  }
+
+  private readonly ruleFilesByRoot = new Map<string, Map<string, IndexRuleFiles>>();
+
   private async traversePaths(
     currentPath: string,
     rootPath: string,
     out: string[],
     depth: number,
     walk: WalkState,
-    stack: IgnoreStack = []
+    stack: IgnoreStack = [],
+    overridden?: Set<string>
   ): Promise<void> {
     if (depth >= CodeScanner.MAX_DEPTH) return;
     if (this.isProtected(currentPath)) return;
@@ -1116,14 +1283,24 @@ export class CodeScanner {
         const stats = lstats;
 
         if (stats.isDirectory()) {
-          if (this.ignoredByStack(relativePath, true, localStack)) continue; // .gitignore/.mnemonikignore
+          // .gitignore/.mnemonikignore, then soft-default directories
+          if (this.skipDirectory(relativePath, localStack)) continue;
           if (await isGitBoundary(fullPath)) continue; // another repo's subtree
-          await this.traversePaths(fullPath, rootPath, out, depth + 1, walk, localStack);
+          await this.traversePaths(
+            fullPath,
+            rootPath,
+            out,
+            depth + 1,
+            walk,
+            localStack,
+            overridden
+          );
         } else if (stats.isFile()) {
-          if (this.ignoredByStack(relativePath, false, localStack)) continue;
-          if (this.isChunkable(fullPath, relativePath)) {
-            out.push(relativePath);
-          }
+          if (!this.isChunkable(fullPath, relativePath)) continue;
+          const decision = this.classifyEntry(relativePath, localStack);
+          if (!decision?.include) continue;
+          out.push(relativePath);
+          if (decision.override) overridden?.add(relativePath);
         }
       } catch (error) {
         walk.complete = false;
@@ -1141,11 +1318,17 @@ export class CodeScanner {
    * but yielded no chunk because each section fell under `minChunkSize`. The
    * daemon still pushes such a file's whole content: it is a doc the project
    * owns, and it would otherwise never reach the server.
+   *
+   * `excluded`, when given, receives every chunkable file a soft default
+   * excluded (indexClassifier.ts), by path or by content. The daemon tells the
+   * server about content exclusions so the server can retire what an earlier
+   * version of the file left in the index.
    */
   async scanFiles(
     filePaths: string[],
     rootPath: string,
-    chunkless?: string[]
+    chunkless?: string[],
+    excluded?: IndexExcludedFile[]
   ): Promise<CodeChunk[]> {
     rootPath = await this.canonicalRoot(rootPath);
     const chunks: CodeChunk[] = [];
@@ -1160,15 +1343,6 @@ export class CodeScanner {
         if (this.shouldIgnore(fileRel)) {
           continue;
         }
-        // Honor .gitignore/.mnemonikignore on the incremental path too, by
-        // reconstructing this file's ancestor ignore-layer stack (memoized
-        // per directory across the batch).
-        if (rootPath && !fileRel.startsWith('..')) {
-          const stack = await this.buildIgnoreStackForFile(fileRel, rootPath, ignoreCache, walk);
-          if (this.ignoredByStack(fileRel, false, stack)) {
-            continue;
-          }
-        }
         // Explicit file lists arrive from watcher events, which can race a
         // worktree/nested-clone appearing - apply the same nested-git-boundary
         // rule the walks use (an ancestor between root and the file carrying
@@ -1176,16 +1350,36 @@ export class CodeScanner {
         if (rootPath && (await this.insideNestedGitBoundary(filePath, rootPath))) {
           continue;
         }
+        if (!this.isChunkable(filePath, fileRel)) continue;
 
-        if (this.isChunkable(filePath, fileRel)) {
-          const fileChunks = await this.parseFile(
-            filePath,
-            rootPath || filePath,
-            undefined,
-            chunkless
-          );
-          chunks.push(...fileChunks);
+        // Honor .gitignore/.mnemonikignore/.gitattributes on the incremental
+        // path too, by reconstructing this file's ancestor rule stack (memoized
+        // per directory across the batch), and judge it exactly as the walks do.
+        let applyContentRules = true;
+        if (rootPath && !fileRel.startsWith('..')) {
+          const stack = await this.buildIgnoreStackForFile(fileRel, rootPath, ignoreCache, walk);
+          const decision = this.classifyEntry(fileRel, stack);
+          if (!decision) continue;
+          if (!decision.include) {
+            excluded?.push({
+              path: fileRel,
+              reason: decision.reason,
+              rule: decision.rule,
+              stage: 'path',
+            });
+            continue;
+          }
+          applyContentRules = !decision.override;
         }
+
+        const fileChunks = await this.parseFile(
+          filePath,
+          rootPath || filePath,
+          undefined,
+          chunkless,
+          { applyContentRules, ...(excluded ? { excluded } : {}) }
+        );
+        chunks.push(...fileChunks);
       } catch (error) {
         logDebug('Error scanning file', { filePath, error });
       }
@@ -1248,7 +1442,8 @@ export class CodeScanner {
     depth: number,
     walk: WalkState,
     stack: IgnoreStack = [],
-    chunkless?: string[]
+    chunkless?: string[],
+    excluded?: IndexExcludedFile[]
   ): Promise<void> {
     // Prevent infinite recursion
     if (depth >= CodeScanner.MAX_DEPTH) {
@@ -1303,7 +1498,8 @@ export class CodeScanner {
         const stats = lstats;
 
         if (stats.isDirectory()) {
-          if (this.ignoredByStack(relativePath, true, localStack)) continue; // .gitignore/.mnemonikignore
+          // .gitignore/.mnemonikignore, then soft-default directories
+          if (this.skipDirectory(relativePath, localStack)) continue;
           if (await isGitBoundary(fullPath)) continue; // another repo's subtree
           await this.traverseDirectory(
             fullPath,
@@ -1312,14 +1508,18 @@ export class CodeScanner {
             depth + 1,
             walk,
             localStack,
-            chunkless
+            chunkless,
+            excluded
           );
         } else if (stats.isFile()) {
-          if (this.ignoredByStack(relativePath, false, localStack)) continue;
-          if (this.isChunkable(fullPath, relativePath)) {
-            const fileChunks = await this.parseFile(fullPath, rootPath, stats, chunkless);
-            chunks.push(...fileChunks);
-          }
+          if (!this.isChunkable(fullPath, relativePath)) continue;
+          const decision = this.classifyEntry(relativePath, localStack);
+          if (!decision?.include) continue;
+          const fileChunks = await this.parseFile(fullPath, rootPath, stats, chunkless, {
+            applyContentRules: !decision.override,
+            ...(excluded ? { excluded } : {}),
+          });
+          chunks.push(...fileChunks);
         }
       } catch (error) {
         walk.complete = false;
@@ -1377,7 +1577,10 @@ export class CodeScanner {
     filePath: string,
     rootPath: string,
     enumerated?: Stats,
-    chunkless?: string[]
+    chunkless?: string[],
+    contentGate: { applyContentRules: boolean; excluded?: IndexExcludedFile[] } = {
+      applyContentRules: true,
+    }
   ): Promise<CodeChunk[]> {
     try {
       // Check file size before reading to avoid memory issues
@@ -1400,18 +1603,39 @@ export class CodeScanner {
         FILE_OP_TIMEOUT_MS,
         `readFile timed out: ${filePath}`
       );
+      const relativePath = relative(rootPath, filePath);
+      // Content soft defaults (binary, generated header, minified, data dump),
+      // read from the SCRUBBED text because that is what the server receives
+      // and re-judges - the two sides must not straddle a threshold. A NUL in
+      // the first 8,000 bytes now means "binary, skip" (git's heuristic)
+      // instead of strip-and-index.
+      if (contentGate.applyContentRules) {
+        const byContent = softDefaultForContent(relativePath, scrubSecrets(content));
+        if (byContent) {
+          contentGate.excluded?.push({
+            path: relativePath.split(sep).join('/'),
+            reason: byContent.reason,
+            rule: byContent.rule,
+            stage: 'content',
+          });
+          logDebug('Skipping file excluded by a content soft default', {
+            filePath: relativePath,
+            ...byContent,
+          });
+          return [];
+        }
+      }
       // NUL sanitation (ingestion boundary, daemon side): Postgres `text`
       // columns reject the literal NUL byte (U+0000) - a single one aborts
-      // the whole INSERT/UPDATE. Some source files legitimately contain
-      // them. Strip before any chunking so every chunk's content, its
-      // contentHash, and its embedding all agree - mirrors the server-side
-      // strip in ProjectManager (kept as defense in depth for content that
-      // reaches the server via an older scanner) but doing it here means
-      // the daemon never even transmits the NUL bytes.
+      // the whole INSERT/UPDATE. Text that is otherwise included can still
+      // carry a stray NUL past the binary window. Strip before any chunking so
+      // every chunk's content, its contentHash, and its embedding all agree -
+      // mirrors the server-side strip in ProjectManager (kept as defense in
+      // depth for content that reaches the server via an older scanner) but
+      // doing it here means the daemon never even transmits the NUL bytes.
       if (content.includes('\0')) {
         content = content.replace(/\0/g, '');
       }
-      const relativePath = relative(rootPath, filePath);
       const language = this.detectLanguage(filePath);
 
       // Try to extract functions/classes

package/src/index.ts

@@ -26,7 +26,41 @@ export {
   logAstCapabilityOnce,
   type CodeChunk,
   type ScanOptions,
+  type IndexExcludedFile,
+  type IndexRuleFiles,
 } from './codeScanner.js';
+export {
+  INDEX_EXCLUSION_REASONS,
+  BINARY_SNIFF_BYTES,
+  MINIFIED_AVERAGE_LINE_LENGTH,
+  MINIFIED_LONG_LINE_BYTES,
+  DATA_DUMP_INSERT_THRESHOLD,
+  NESTED_IGNORE_TAG,
+  GITATTRIBUTES_TAG,
+  classifyForIndex,
+  softDefaultForPath,
+  softDefaultForDirectory,
+  softDefaultForContent,
+  parseGitattributes,
+  linguistAttributesFor,
+  gitattributesCanReinclude,
+  indexExclusionLabel,
+  indexExclusionNotice,
+  isIndexExclusionReason,
+  encodeNestedIgnoreLine,
+  encodeGitattributesLine,
+  decodeIndexRules,
+  type IndexExclusionReason,
+  type IndexExclusion,
+  type IndexClassification,
+  type IgnoreFileVerdict,
+  type GitattributesLayer,
+  type GitattributesRule,
+  type LinguistAttributes,
+  type ClassifyForIndexOptions,
+  type DecodedIndexRules,
+} from './indexClassifier.js';
+export { makeIndexJudge, type IndexJudge, type IndexJudgement } from './indexJudge.js';
 export {
   AST_LANGUAGE_IDS,
   QUERY_CHAIN,