@mnemonik/shared 7.154.2 → 7.155.2

package/src/codeScanner.ts

@@ -2,16 +2,12 @@
  * Code Scanner - Parse and chunk source files for embedding
  */
 
-import { readdir, readFile, open, lstat, realpath } from 'fs/promises';
-import { constants, type Stats } from 'fs';
-import { join, relative, extname, sep, resolve, dirname } from 'path';
+import { readdir, lstat } from 'fs/promises';
+import { type Stats } from 'fs';
+import { join, relative, extname, sep } from 'path';
 import { createHash } from 'crypto';
-import ignore, { type Ignore } from 'ignore';
 import { debug as logDebug, info as logInfo, warn as logWarn } from './logger.js';
-import { withTimeout } from './asyncUtils.js';
 import { scrubSourceSecrets } from './sourceRedaction.js';
-import { isProtectedLocalPath, protectedLocalPaths } from './protectedPaths.js';
-import { readIdentityFile } from './projectIdentityFile.js';
 import {
   MAX_AST_PARSE_BYTES,
   MAX_SIGNATURE_CHARS,
@@ -19,23 +15,66 @@ import {
   type AstChunk,
 } from './ast/astChunker.js';
 import { astArtifactReport, resolveAstLanguage } from './ast/grammars.js';
+import { softDefaultForContent } from './indexClassifier.js';
 import {
-  classifyForIndex,
-  gitattributesCanReinclude,
-  parseGitattributes,
-  softDefaultForContent,
-  softDefaultForDirectory,
-  type GitattributesLayer,
-  type GitattributesRule,
-  type IgnoreFileVerdict,
-  type IndexClassification,
-  type IndexExclusionReason,
-} from './indexClassifier.js';
+  FileSelector,
+  DEFAULT_IGNORE_PATTERNS,
+  fileOp,
+  markUnreadable,
+  enterDirectory,
+  DEFAULT_INCLUDE_EXTENSIONS,
+  isCursorRulePath,
+  languageForExtension,
+  AUTHORITY_FILE_MATCHERS,
+  type FileSelectionOptions,
+  type WalkState,
+  type IndexExcludedFile,
+  type IgnoreLayer,
+  type IgnoreStack,
+} from './fileSelection.js';
+// The selection moved to fileSelection.ts; its names stay exported from here.
+export {
+  FileSelector,
+  DEFAULT_IGNORE_PATTERNS,
+  fileOp,
+  statTuple,
+  BUILT_IN_IGNORE_DIRS,
+  isSecretFile,
+  DEFAULT_INCLUDE_EXTENSIONS,
+  isCursorRulePath,
+  languageForExtension,
+  XCODE_AUTHORITY_PATH,
+  isXcodeAuthorityPath,
+  CMAKE_LISTS_PATH,
+  isCmakeListsPath,
+  isAuthorityOnlyPath,
+  AUTHORITY_FILE_MATCHERS,
+  FIXTURE_PATH_RE,
+  isFixturePath,
+  makeIgnoreMatcher,
+  isGitBoundary,
+  isProjectBoundary,
+  rootProjectId,
+  isLinkedWorktreeCheckout,
+  type FileSelectionOptions,
+  type FileStatTuple,
+  type IndexRuleFiles,
+  type IndexExcludedFile,
+} from './fileSelection.js';
 
 /**
- * File operation timeout (5 seconds) to prevent hanging on slow/unresponsive filesystems
+ * The version of the scanner's file hash: the SHA-256 of the scrubbed text
+ * (`scrubSourceSecrets` over the file with NULs removed) and the content soft
+ * default judged on it. The daemon caches that hash per file (size, mtime,
+ * ctime, inode), so a restart or a release does not re-read and re-scrub every
+ * file. Bump this number whenever redaction or the content soft defaults change
+ * what they produce: every machine then re-reads and re-scrubs every file once,
+ * which is exactly what a redaction fix needs. Do NOT bump it for a release that
+ * leaves both unchanged; a needless bump costs every user a full re-read.
+ * tests/ScrubVersion.test.ts pins the scrubber's output on a fixture next to
+ * this number, so a scrubber change that forgets the bump fails there.
  */
-const FILE_OP_TIMEOUT_MS = 5000;
+export const SCRUB_VERSION = 1;
 
 /**
  * The one file-size ceiling in Mnemonik. A file at or below this is scanned,
@@ -93,11 +132,10 @@ export interface CodeChunk {
   };
 }
 
-export interface ScanOptions {
+/** CodeScanner's options: the file selection's (fileSelection.ts) and its chunking. */
+export interface ScanOptions extends FileSelectionOptions {
   maxChunkSize?: number; // characters
   minChunkSize?: number;
-  ignorePatterns?: string[];
-  includeExtensions?: string[];
   /**
    * Byte ceiling above which a file is chunked heuristically instead of parsed.
    * Defaults to `MAX_AST_PARSE_BYTES`; exposed so a caller (and the routing
@@ -105,717 +143,17 @@ export interface ScanOptions {
    * mock.
    */
   maxAstParseBytes?: number;
-  /** Additional protected paths for isolated homes/tests; defaults can only be extended. */
-  protectedPaths?: readonly string[];
-}
-
-/**
- * Mutable per-walk state threaded through the recursive traversals.
- * `complete` starts true and is latched false by any caught fs error
- * (readdir failure, per-entry lstat/realpath/stat throw) - meaning the
- * result may be missing entries that still exist on disk. Callers that
- * derive *removals* from a walk must treat an incomplete walk as
- * untrustworthy; adds/changes are unaffected.
- */
-interface WalkState {
-  complete: boolean;
-  /**
-   * When present, every rule file the walk reads is recorded here by
-   * directory, so the daemon can upload the lines it actually applied.
-   */
-  rules?: Map<string, IndexRuleFiles>;
-}
-
-/** The rule-file lines one directory contributed to a walk. */
-export interface IndexRuleFiles {
-  /** .gitignore + .mnemonikignore lines, unioned, in read order. */
-  ignore: string[];
-  /** .gitattributes lines, in read order. */
-  attributes: string[];
-}
-
-/**
- * A file a walk declined to index because a soft default excluded it.
- * `stage: 'content'` means the file was read and its content decided.
- */
-export interface IndexExcludedFile {
-  /** Root-relative POSIX path, spelled as a chunk's `filePath`. */
-  path: string;
-  reason: IndexExclusionReason;
-  rule: string;
-  stage: 'path' | 'content';
-}
-
-/**
- * Directory basenames the scanner never indexes, across every ecosystem -
- * generated output, package-manager/build caches, and vendored deps. SINGLE
- * SOURCE OF TRUTH: the scanner package's discovery + watch walks import this
- * (`packages/scanner/src/{discovery,watcher}.ts`) and `src/core/utils.ts`
- * derives from it, so every ignore surface agrees instead of drifting (the
- * pre-2026-07 state had four divergent copies). Add only UNAMBIGUOUS names -
- * a project's own `.gitignore` / `.mnemonikignore` covers the ambiguous,
- * project-specific cases (`vendor/`, custom dirs) hierarchically.
- */
-export const BUILT_IN_IGNORE_DIRS: readonly string[] = [
-  // JS / web
-  'node_modules',
-  '.next',
-  '.nuxt',
-  '.output',
-  'dist',
-  'build',
-  'coverage',
-  '.turbo',
-  '.vercel',
-  '.svelte-kit',
-  '.angular',
-  '.astro',
-  '.parcel-cache',
-  'bower_components',
-  // Python
-  'venv',
-  '.venv',
-  'env',
-  '__pycache__',
-  '.tox',
-  '.mypy_cache',
-  '.pytest_cache',
-  '.ruff_cache',
-  // Native mobile - iOS / Android / React Native / Flutter (2026-07-11 Pods incident)
-  'Pods',
-  'DerivedData',
-  '.gradle',
-  '.expo',
-  '.dart_tool',
-  // Infra / cloud
-  '.terraform',
-  'cdk.out',
-  '.serverless',
-  // Build systems, caches, VCS
-  'target',
-  'bin',
-  'obj',
-  '.cache',
-  '.git',
-  '.svn',
-  '.hg',
-  // The coding agent's captured shell environment, written as .sh files.
-  // Harmless while only source extensions were collected; the moment shell
-  // scripts became collectable these scanned as project code, which would put
-  // an agent's own shell state into the user's searchable memory.
-  //
-  // Scoped to this one directory name deliberately. `.claude/` at large holds
-  // real project content - `.claude/rules.md` is indexed on purpose
-  // (tests/CodeScannerIgnorePatterns.test.ts), and only `.claude/worktrees/`
-  // is suppressed, by the generic nested-git-boundary rule.
-  'shell-snapshots',
-];
-
-/**
- * Non-directory built-in ignore patterns (files, globs, and the segment-
- * anchored fixture rule), combined with BUILT_IN_IGNORE_DIRS to form
- * DEFAULT_OPTIONS.ignorePatterns. See `shouldIgnore` for match semantics.
- *
- * `/tests/fixtures/*`: leading '/' = segment-anchored - matches tests/fixtures/
- * at the path root or immediately after a '/', so nested packages/x/tests/
- * fixtures/ are excluded but integration-tests/fixtures/ is NOT. Fixture
- * snapshot dirs were being indexed as real source and outranking the genuine
- * file in code_search. (Subprojects and linked worktrees are excluded
- * generically by the isProjectBoundary rule below, not a path convention.)
- */
-const BUILT_IN_IGNORE_FILE_PATTERNS: readonly string[] = [
-  // NOTE: the `.env` family and every other secret-bearing FILE KIND lives in
-  // `isSecretFile` below, not here. This glob engine cannot express negation
-  // (`.env.example` must stay collectable) and its unanchored substring
-  // matching would make `.env.*` also swallow `.environment.ts`.
-  //
-  // `/tests/fixtures/*` used to live here. It is now a soft default in
-  // indexClassifier.ts (test fixtures, R2), which a project can re-include.
-  '.DS_Store',
-  '*.log',
-  '*.lock',
-  'package-lock.json',
-  'yarn.lock',
-  '*.min.js',
-  '*.min.css',
-  '*.bundle.js',
-  '*.legacy.js',
-  '*.map',
-];
-
-/**
- * Files whose CONTENT is a credential rather than code. Today these are
- * excluded only INCIDENTALLY - `.pem`, `.key` and friends carry no
- * allowlisted extension, so nothing chunks them. That protection evaporates
- * the moment the extension allowlist widens (as it did for `.sh`/`.lua`), so
- * the exclusion is made explicit here and becomes a defended invariant:
- * unlike `ignorePatterns`, this predicate is NOT caller-overridable.
- *
- * Expressed as a predicate rather than glob strings because `shouldIgnore`'s
- * engine has no negation and matches globs as unanchored substrings - neither
- * property can state "every `.env.*` EXCEPT `.env.example`".
- *
- * Precision notes (over-exclusion is a fidelity failure in its own right):
- * - `.env` family: `/^\.env(\..+)?$/` matches `.env`, `.env.local`,
- *   `.env.production` - and deliberately NOT `.envrc` or `.environment.ts`.
- * - `.env.example` is carved out: it is authority-collected on purpose
- *   (AUTHORITY_FILE_MATCHERS) and holds placeholder values, not secrets.
- *   It is the doc-truth env_vars authority; excluding it would blind that
- *   extractor exactly as the old `process.env` over-scrub once did.
- * - `credentials` is matched ONLY directly inside a dot-directory
- *   (`.aws/credentials`, `.docker/credentials`). A bare `credentials/` is a
- *   real, common SOURCE directory name (gRPC, auth libraries) and
- *   `shouldIgnore` cannot tell a file from a directory at its call sites -
- *   an unqualified rule would silently delete that subtree from the index.
- * - `.tfvars` is excluded whole rather than per-basename: the credential file
- *   is conventionally `terraform.tfvars` but the name is free-form, and the
- *   variables that are NOT secret are visible in `.tf` anyway.
- */
-const SECRET_FILE_PLACEHOLDER_BASENAMES: ReadonlySet<string> = new Set(['.env.example']);
-
-const SECRET_FILE_BASENAME_PATTERNS: readonly RegExp[] = [
-  /^\.env(\..+)?$/i, // .env, .env.local, .env.production.local
-  /^id_(?:rsa|dsa|ecdsa|ed25519)/i, // ssh private keys (and .pub siblings)
-  /^\.(?:npmrc|netrc|pgpass)$/i, // registry / ftp / postgres password files
-  /^kubeconfig$/i,
-  /\.(?:pem|key|p12|pfx|keystore|jks|kubeconfig)$/i,
-  // Terraform variable files: `terraform.tfvars` is the conventional home for
-  // provider credentials, and unlike the entries above nothing else was
-  // keeping it out - it is excluded here rather than merely left off
-  // `DEFAULT_INCLUDE_EXTENSIONS`, so a future widening of the allowlist cannot
-  // quietly opt a project into embedding its own cloud keys.
-  /\.tfvars(\.json)?$/i,
-];
-
-/**
- * True when `relPath` names a file whose contents are credentials. Applied
- * by `shouldIgnore` (every walker: directory scan, explicit file lists, and
- * the authority-file walk) and by `makeIgnoreMatcher` (server-side backstop
- * for older or misbehaving daemons). Accepts OS-native or POSIX separators.
- */
-export function isSecretFile(relPath: string): boolean {
-  if (!relPath) return false;
-  const segments = relPath.split(sep).join('/').split('/');
-  const base = segments[segments.length - 1] ?? '';
-  if (!base || SECRET_FILE_PLACEHOLDER_BASENAMES.has(base)) return false;
-  if (SECRET_FILE_BASENAME_PATTERNS.some((re) => re.test(base))) return true;
-  const parent = segments[segments.length - 2];
-  return base === 'credentials' && parent !== undefined && parent.startsWith('.');
-}
-
-/**
- * Per-directory ignore files the walkers honor, unioned. `.gitignore` is the
- * project's own declaration of generated/vendored paths; `.mnemonikignore`
- * (same gitignore syntax) is the user's Mnemonik-specific "never index this"
- * list (e.g. `secrets/`). Matched hierarchically via the `ignore` package.
- */
-const IGNORE_FILE_NAMES = ['.gitignore', '.mnemonikignore'] as const;
-
-/** Per-directory attributes file: linguist overrides and binary marks (indexClassifier.ts). */
-const ATTRIBUTES_FILE_NAME = '.gitattributes';
-
-/**
- * Every file type the scanner will chunk, by extension. Matched
- * case-INSENSITIVELY (see `isIncludedExtension`), so lowercase spellings here
- * also cover `.R`, `.SQL`, `.PS1` and the uppercase `.C`/`.H` of older trees.
- *
- * Exported because this list IS the product's language coverage: a project
- * written in something absent from it indexes zero code, and `code_search`
- * reports that as "no indexed matches" rather than "I do not read this
- * language" - invisible to the developer and to us. Coverage is a separate
- * dial from parse quality: a language with no structured extractor falls
- * through to `chunkRaw`, which is what Go, Java and C already get in
- * production, and crude chunks beat no chunks by an enormous margin.
- *
- * Data formats (`.json`, `.yaml`, `.toml`, `.lock`) are deliberately absent:
- * the ones that carry meaning are already collected verbatim by
- * `AUTHORITY_FILE_MATCHERS`, and blanket-indexing the extension would pull in
- * lockfiles and generated output. `.tfvars` is absent for a different reason -
- * `terraform.tfvars` is a conventional home for provider credentials and
- * `isSecretFile` now excludes it outright.
- */
-export const DEFAULT_INCLUDE_EXTENSIONS: readonly string[] = [
-  '.ts',
-  '.tsx',
-  '.js',
-  '.jsx',
-  '.py',
-  '.rs',
-  '.go',
-  '.java',
-  '.c',
-  '.cpp',
-  '.h',
-  '.cs',
-  '.rb',
-  '.php',
-  '.swift',
-  '.kt',
-  // Shell and Lua are the working languages of whole real projects -
-  // deployment tooling, container entrypoints, imapfilter/nginx/redis
-  // configuration. Omitting them meant such a project indexed ZERO code and
-  // code_search could not answer anything about it, while reporting that as
-  // "no indexed matches" rather than as missing coverage.
-  '.sh',
-  '.bash',
-  '.zsh',
-  '.lua',
-  '.md',
-
-  // -- Variants of languages already on the list. `.mjs`/`.cjs`/`.mts`/`.cts`/
-  // `.pyi` inherit their language string from a listed extension, so their
-  // absence was oversight rather than policy. The C++ spellings are a choice,
-  // not an inheritance: `.h` stays 'c' (C headers dominate, and reading a C++
-  // header as C is the safer default), while the unambiguously-C++ spellings
-  // resolve to 'cpp'.
-  '.mjs',
-  '.cjs',
-  '.mts',
-  '.cts',
-  '.pyi',
-  '.hpp',
-  '.hh',
-  '.hxx',
-  '.cc',
-  '.cxx',
-  '.kts',
-
-  // -- Languages the scanner could not read at all.
-  '.dart',
-  '.m',
-  '.mm',
-  '.scala',
-  '.sc',
-  '.ex',
-  '.exs',
-  '.erl',
-  '.hrl',
-  '.hs',
-  '.jl',
-  '.ml',
-  '.mli',
-  '.clj',
-  '.cljs',
-  '.cljc',
-  '.groovy',
-  '.gradle',
-  '.ps1',
-  '.psm1',
-  '.pl',
-  '.pm',
-  '.r',
-  '.sol',
-  '.zig',
-  '.vue',
-  '.svelte',
-
-  // -- Infrastructure and schema DSLs, where real logic lives and where "how
-  // is this deployed?" and "what does this table hold?" go unanswered today.
-  '.tf',
-  '.sql',
-  '.proto',
-  '.graphql',
-  '.gql',
-  '.cmake',
-  '.nix',
-  '.bzl',
-];
-
-/**
- * Cursor's rule files: `.cursorrules`, and any `.mdc` under `.cursor/rules/`.
- * Cursor is a launch host, so these are instruction files, the same class as
- * CLAUDE.md and AGENTS.md. Neither spelling can come in through
- * `DEFAULT_INCLUDE_EXTENSIONS`: `.cursorrules` has no extension (`extname`
- * answers ''), and `.mdc` is not a format this scanner reads anywhere else. So
- * both are admitted by path at the chunking gate (`isChunkable`) and chunked as
- * Markdown. They are not `.md` paths, so their whole-file content lands in
- * project_source_content.
- *
- * Matched at any depth, because Cursor also reads nested `.cursor/rules`
- * directories in a monorepo. Case-sensitive, as Cursor spells them. Accepts
- * OS-native or POSIX separators.
- */
-export function isCursorRulePath(relPath: string): boolean {
-  if (!relPath) return false;
-  const posix = relPath.split(sep).join('/');
-  return /(^|\/)\.cursorrules$/.test(posix) || /(^|\/)\.cursor\/rules\/.+\.mdc$/.test(posix);
-}
-
-/**
- * Extension -> language string, the value carried on every chunk and on the
- * wire (`/scan/push` accepts any non-empty string up to 50 chars).
- *
- * A string no grammar claims is correct and expected: the AST layer resolves
- * such a language to `null` and the heuristic chunker takes over. Keep this in
- * sync with `DEFAULT_INCLUDE_EXTENSIONS` - an allowlisted extension that lands
- * on 'unknown' still gets chunked, but nothing downstream can reason about it.
- *
- * Two extensions are genuinely ambiguous and are resolved rather than fudged:
- * `.m` is Objective-C here, not MATLAB, because a repo carrying `.m` alongside
- * `.h`/`.mm` is overwhelmingly an Apple-platform project; `.pl` is Perl, not
- * Prolog, on the same frequency argument.
- */
-const EXTENSION_LANGUAGES: Readonly<Record<string, string>> = {
-  '.ts': 'typescript',
-  '.tsx': 'typescript',
-  '.mts': 'typescript',
-  '.cts': 'typescript',
-  '.js': 'javascript',
-  '.jsx': 'javascript',
-  '.mjs': 'javascript',
-  '.cjs': 'javascript',
-  '.py': 'python',
-  '.pyi': 'python',
-  '.rs': 'rust',
-  '.go': 'go',
-  '.java': 'java',
-  '.c': 'c',
-  '.h': 'c',
-  '.cpp': 'cpp',
-  '.cc': 'cpp',
-  '.cxx': 'cpp',
-  '.hpp': 'cpp',
-  '.hh': 'cpp',
-  '.hxx': 'cpp',
-  '.cs': 'csharp',
-  '.rb': 'ruby',
-  '.php': 'php',
-  '.swift': 'swift',
-  '.kt': 'kotlin',
-  '.kts': 'kotlin',
-  '.sh': 'shell',
-  '.bash': 'shell',
-  '.zsh': 'shell',
-  '.lua': 'lua',
-  '.md': 'markdown',
-  '.dart': 'dart',
-  '.m': 'objc',
-  '.mm': 'objc',
-  '.scala': 'scala',
-  '.sc': 'scala',
-  '.ex': 'elixir',
-  '.exs': 'elixir',
-  '.erl': 'erlang',
-  '.hrl': 'erlang',
-  '.hs': 'haskell',
-  '.jl': 'julia',
-  '.ml': 'ocaml',
-  '.mli': 'ocaml',
-  '.clj': 'clojure',
-  '.cljs': 'clojure',
-  '.cljc': 'clojure',
-  '.groovy': 'groovy',
-  '.gradle': 'groovy',
-  '.ps1': 'powershell',
-  '.psm1': 'powershell',
-  '.pl': 'perl',
-  '.pm': 'perl',
-  '.r': 'r',
-  '.sol': 'solidity',
-  '.zig': 'zig',
-  '.vue': 'vue',
-  '.svelte': 'svelte',
-  '.tf': 'terraform',
-  '.sql': 'sql',
-  '.proto': 'protobuf',
-  '.graphql': 'graphql',
-  '.gql': 'graphql',
-  '.cmake': 'cmake',
-  '.nix': 'nix',
-  '.bzl': 'starlark',
-};
-
-/**
- * Language string for a file path or a bare extension, `'unknown'` when the
- * extension is unmapped. A free function rather than a method because callers
- * that never scan anything (AST grammar resolution, server-side symbol
- * preference) need the same answer without constructing a scanner.
- */
-export function languageForExtension(filePathOrExt: string): string {
-  if (isCmakeListsPath(filePathOrExt)) return 'cmake';
-  // `extname` FIRST. A dotfile that carries a real extension ('.eslintrc.js',
-  // '.mocharc.cjs', '.prettierrc.ts') starts with '.' and contains no '/', so
-  // a bare-extension-first reading swallowed the whole name and answered
-  // 'unknown' - while the same file spelled 'src/.eslintrc.js' answered
-  // 'javascript'. Those extensions are allowlisted, so the files are indexed
-  // and reach the server with exactly the root-relative spelling that failed.
-  const fromPath = EXTENSION_LANGUAGES[extname(filePathOrExt).toLowerCase()];
-  if (fromPath !== undefined) return fromPath;
-  // Fallback: the argument IS the extension ('.dart'), for which `extname`
-  // returns ''. Every key contains a leading dot and no separator, so a real
-  // path can never collide here.
-  return EXTENSION_LANGUAGES[filePathOrExt.toLowerCase()] ?? 'unknown';
-}
-
-/**
- * THE definition of "this path is a SQL migration the schema_columns authority
- * collects verbatim". One predicate, referenced by both halves of the deal -
- * `AUTHORITY_FILE_MATCHERS` (collect it) and `isAuthorityOnlyPath` (therefore
- * do not chunk it) - because two hand-written regexes drifted once already and
- * the failure is silent in both directions.
- *
- * Root-anchored and CASE-SENSITIVE on purpose: it mirrors the server-side
- * extractor, which does `listFiles('migrations/')` (LIKE 'migrations/%') then
- * `endsWith('.sql')`, both case-sensitive. `Migrations/001.sql` (the .NET/EF
- * Core spelling) and `migrations/002.SQL` are NOT collected, so they must not
- * be suppressed from chunking either - that would index them nowhere. Same
- * reason a nested `packages/x/migrations/y.sql` is left alone.
- */
-const isMigrationSqlAuthorityPath = (posixRelPath: string): boolean =>
-  /^migrations\/.*\.sql$/.test(posixRelPath);
-
-/** Content-only build authority, shared by scanner collection and server admission. */
-export const XCODE_AUTHORITY_PATH = /(?:^|\/)[^/]+\.xcodeproj\/project\.pbxproj$|\.xcconfig$/;
-export const isXcodeAuthorityPath = (path: string): boolean => XCODE_AUTHORITY_PATH.test(path);
-/** Exact CMake build entrypoint; ordinary .txt files remain outside collection. */
-export const CMAKE_LISTS_PATH = /(?:^|\/)CMakeLists\.txt$/;
-export const isCmakeListsPath = (path: string): boolean => CMAKE_LISTS_PATH.test(path);
-
-/**
- * Predicates for paths whose verbatim content is ALREADY shipped by
- * `collectAuthorityFiles` and that carry no additional value as embedded code
- * chunks. Checked at the extension gate rather than in `shouldIgnore`, because
- * `shouldIgnore` also guards the authority walk and must keep letting these
- * through.
- *
- * `migrations/**.sql` is the live case: adding `.sql` to the allowlist without
- * this exclusion would dual-collect every migration - once verbatim, once
- * chunked and embedded. On this repo alone that is 208 files of append-only
- * DDL (140 forward, the rest rollback/manual), embedded to answer questions
- * the authority path already answers exactly.
- *
- * INVARIANT: every predicate here must also appear in
- * `AUTHORITY_FILE_MATCHERS`, so no path can be excluded from chunking unless
- * the authority path definitely collects it.
- */
-const AUTHORITY_ONLY_PATH_PREDICATES: readonly ((posixRelPath: string) => boolean)[] = [
-  isMigrationSqlAuthorityPath,
-  isXcodeAuthorityPath,
-  isCmakeListsPath,
-];
-
-/**
- * True when `relPath` is collected verbatim as authority content and must not
- * also be chunked. Accepts OS-native or POSIX separators.
- */
-export function isAuthorityOnlyPath(relPath: string): boolean {
-  if (!relPath) return false;
-  const posix = relPath.split(sep).join('/');
-  return AUTHORITY_ONLY_PATH_PREDICATES.some((matches) => matches(posix));
 }
 
 const DEFAULT_OPTIONS: Required<ScanOptions> = {
   maxChunkSize: 8000, // ~2000 tokens
   minChunkSize: 100,
-  ignorePatterns: [...BUILT_IN_IGNORE_DIRS, ...BUILT_IN_IGNORE_FILE_PATTERNS],
+  ignorePatterns: [...DEFAULT_IGNORE_PATTERNS],
   includeExtensions: [...DEFAULT_INCLUDE_EXTENSIONS],
   maxAstParseBytes: MAX_AST_PARSE_BYTES,
   protectedPaths: [],
 };
 
-/**
- * Matchers for authority manifest / config / CI files whose verbatim content
- * is collected by `collectAuthorityFiles` and pushed to the server for
- * doc-truth Fingerprint parsing. Operates on relative, forward-slash paths.
- */
-export const AUTHORITY_FILE_MATCHERS: Array<(relPath: string) => boolean> = [
-  (p) => p === 'package.json',
-  (p) => /^(packages|apps|services|tools)\/[^/]+\/package\.json$/.test(p),
-  (p) => p === 'tsconfig.json' || /^tsconfig\.[^/]+\.json$/.test(p),
-  (p) => p === 'pyproject.toml',
-  (p) => p === 'requirements.txt',
-  (p) => p === 'setup.py',
-  (p) => p === 'Cargo.toml',
-  (p) => p === 'Gemfile',
-  (p) => p === 'Makefile',
-  (p) => p === '.env.example',
-  (p) => /^\.github\/workflows\/[^/]+\.(yml|yaml)$/.test(p),
-  // SQL migrations: the schema_columns authority extractor reads every .sql
-  // under `migrations/` (listFiles('migrations/') -> LIKE 'migrations/%' then
-  // .endsWith('.sql')). Without collecting these, that authority is empty and
-  // every schema_table_enumeration claim falls to unverifiable. Shared with
-  // `AUTHORITY_ONLY_PATH_PREDICATES` by reference, not by a copied regex, so
-  // collection and the chunking exclusion cannot disagree.
-  isMigrationSqlAuthorityPath,
-  isXcodeAuthorityPath,
-  isCmakeListsPath,
-];
-
-/**
- * Segment-anchored match for `tests/fixtures/` - matches at the path root or
- * immediately after a `/`. Indexing no longer uses it (the fixture rule is the
- * soft default in indexClassifier.ts, which a project can override); it stays
- * for query-time hygiene over rows indexed before that rule existed
- * (`MemoryManager.searchCodePointers`).
- */
-export const FIXTURE_PATH_RE = /(^|\/)tests\/fixtures\//;
-export function isFixturePath(path: string): boolean {
-  return FIXTURE_PATH_RE.test(path);
-}
-
-const BUILT_IN_IGNORE_DIR_SET = new Set<string>(BUILT_IN_IGNORE_DIRS);
-
-/**
- * Server-side ignore matcher for defense-in-depth path dropping on /scan/push,
- * /scan/reconcile, and file_push. Combines the universal BUILT_IN_IGNORE_DIRS
- * (segment-exact) with a project's uploaded .gitignore/.mnemonikignore patterns
- * (gitignore syntax via the `ignore` package). Built once per request and reused
- * across all pushed paths (root-relative, forward-slash). This does NOT replace
- * the daemon's precise hierarchical walk - it backstops it so a buggy, stale, or
- * malicious client cannot index a path the project ignores, making .mnemonikignore
- * a hard guarantee rather than best-effort.
- */
-export function makeIgnoreMatcher(
-  extraPatterns: readonly string[] = []
-): (relPath: string) => boolean {
-  const cleaned = extraPatterns.filter((p) => typeof p === 'string' && p.trim().length > 0);
-  const ig = cleaned.length > 0 ? ignore().add([...cleaned]) : null;
-  return (relPath: string): boolean => {
-    if (!relPath) return false;
-    const norm = relPath.split(sep).join('/');
-    if (norm.split('/').some((s) => BUILT_IN_IGNORE_DIR_SET.has(s))) return true;
-    if (isSecretFile(norm)) return true;
-    if (ig) {
-      try {
-        return ig.ignores(norm);
-      } catch {
-        // `ignore` throws on absolute/`..` paths (safeScanPath already rejects
-        // those upstream); built-ins were already checked, so fail open here.
-        return false;
-      }
-    }
-    return false;
-  };
-}
-
-/**
- * True when `dirPath` is the root of a DIFFERENT repository than the one
- * being scanned: it carries its own `.git` entry. A `.git` FILE marks a
- * linked git worktree (wherever the user keeps them) or a submodule
- * checkout; a `.git` DIRECTORY marks a nested clone. Files under such a
- * boundary belong to that repository's own project identity - scanning them
- * into the parent project is wrong attribution for any codebase, so the
- * walkers skip these subtrees. Convention-free: this is git's own marker,
- * not a host/tool path layout. The scan ROOT is exempt by construction
- * (callers only test child directories).
- */
-export async function isGitBoundary(dirPath: string): Promise<boolean> {
-  try {
-    await withTimeout(
-      lstat(join(dirPath, '.git')),
-      FILE_OP_TIMEOUT_MS,
-      `lstat timed out: ${join(dirPath, '.git')}`
-    );
-    return true;
-  } catch {
-    return false;
-  }
-}
-
-/**
- * How many folders deep a project's scan walks. Discovery searches a project
- * for subprojects to the same depth, so a subproject the scan leaves out is
- * always found and scanned as its own project.
- */
-export const SCANNER_MAX_DEPTH = 10;
-
-/**
- * True when `dirPath` starts a subtree that is not this project's to index.
- * A file belongs to the project of the nearest `.mnemonik.json` above it, so:
- *  - a folder with its own identity file is a subproject, indexed as its own
- *    project, never also into this one. A file naming the enclosing project
- *    (`enclosingProjectId`, a copy) is the same project, and a
- *    configuration-only file (no `projectId`) names none: neither is a
- *    boundary. A file that cannot be read as an identity still is, and so is
- *    a copy in a full clone (its own `.git` DIRECTORY): a second checkout of
- *    the project, whose files would otherwise be indexed twice;
- *  - a linked worktree (a `.git` FILE whose git directory has a `commondir`,
- *    or is missing) is another checkout of a project, indexed from its main
- *    checkout.
- * A nested clone or a submodule without its own `.mnemonik.json` belongs to
- * this project and is indexed with it (its `.git` directory is ignored by the
- * built-in rules). The scan ROOT is exempt by construction (callers only test
- * child directories).
- */
-export async function isProjectBoundary(
-  dirPath: string,
-  enclosingProjectId?: string
-): Promise<boolean> {
-  if (await lstatWithin(join(dirPath, '.mnemonik.json'))) {
-    const identity = await withTimeout(
-      readIdentityFile(dirPath, { configurationOnlyAbsent: true }),
-      FILE_OP_TIMEOUT_MS,
-      `identity read timed out: ${dirPath}`
-    ).catch(() => undefined);
-    if (!identity) return true;
-    if (identity.kind === 'ok') {
-      if (identity.identity.projectId !== enclosingProjectId) return true;
-      if ((await lstatWithin(join(dirPath, '.git')))?.isDirectory()) return true;
-    } else if (identity.kind !== 'absent') return true;
-  }
-  return isLinkedWorktreeCheckout(dirPath);
-}
-
-/**
- * The project a scan root's own identity file names, or undefined: the
- * enclosing project for isProjectBoundary.
- */
-export async function rootProjectId(rootPath: string): Promise<string | undefined> {
-  const identity = await withTimeout(
-    readIdentityFile(rootPath),
-    FILE_OP_TIMEOUT_MS,
-    `identity read timed out: ${rootPath}`
-  ).catch(() => undefined);
-  return identity?.kind === 'ok' ? identity.identity.projectId : undefined;
-}
-
-const lstatWithin = (path: string) =>
-  withTimeout(lstat(path), FILE_OP_TIMEOUT_MS, `lstat timed out: ${path}`).catch(() => undefined);
-
-/**
- * True when `dirPath` is the top of a linked git worktree: its `.git` is a
- * FILE pointing at a git directory that has a `commondir`, or at one that no
- * longer exists (a pruned or moved worktree, or one made on another machine):
- * nothing shows it is this project's own. A submodule's `.git` file points at
- * `.git/modules/<name>`, which exists and has no `commondir`.
- */
-export async function isLinkedWorktreeCheckout(dirPath: string): Promise<boolean> {
-  const marker = await lstatWithin(join(dirPath, '.git'));
-  if (!marker?.isFile()) return false;
-  try {
-    const pointer = /^gitdir:\s*(.+?)\s*$/im.exec(
-      await withTimeout(
-        readFile(join(dirPath, '.git'), 'utf8'),
-        FILE_OP_TIMEOUT_MS,
-        `read timed out: ${join(dirPath, '.git')}`
-      )
-    )?.[1];
-    if (!pointer) return false;
-    const gitDirectory = resolve(dirPath, pointer);
-    if (!(await lstatWithin(gitDirectory))) return true;
-    return !!(await lstatWithin(join(gitDirectory, 'commondir')));
-  } catch {
-    return false;
-  }
-}
-
-/**
- * One directory's compiled ignore-file rules, keyed by its path relative to
- * the scan root (`''` = root). A stack of these, root->leaf, is threaded
- * through the walkers; `ignoredByStack` tests deepest-first so a nested
- * `.gitignore` overrides a shallower one (git precedence).
- */
-interface IgnoreLayer {
-  baseRel: string;
-  /** Compiled .gitignore + .mnemonikignore; null when this directory has neither. */
-  ig: Ignore | null;
-  /** Any `!pattern` line: a path under here may be re-included from a soft default. */
-  hasNegation: boolean;
-  /** Parsed .gitattributes; null when this directory has none. */
-  attributes: GitattributesRule[] | null;
-}
-type IgnoreStack = IgnoreLayer[];
-
 /**
  * Latch for `logAstCapabilityOnce`. A promise, not a boolean: two concurrent
  * callers must both wait on the same report rather than the second returning
@@ -875,389 +213,17 @@ export function logAstCapabilityOnce(): Promise<void> {
   return astCapabilityLog;
 }
 
-export class CodeScanner {
+export class CodeScanner extends FileSelector {
   private options: Required<ScanOptions>;
-  private readonly protectedPaths: readonly string[];
-  /** Cumulative observations since this scanner started, including repeated walks. */
-  readonly skipReasons = { symlink_skipped: 0 };
-
-  private skipSymlink(stats: Stats): boolean {
-    if (!stats.isSymbolicLink()) return false; // Includes Windows junctions.
-    this.skipReasons.symlink_skipped++;
-    return true;
-  }
-
-  /**
-   * `includeExtensions` folded to lowercase for matching. The gate used to
-   * compare `extname()` verbatim while `detectLanguage` lowercased, so a
-   * project spelling its files the canonical way - `.R` for R, `.SQL`/`.PS1`
-   * on Windows, `.C`/`.H` in older C trees - indexed zero of them and nothing
-   * said why.
-   */
-  private readonly includeExtensionSet: ReadonlySet<string>;
 
   constructor(options: ScanOptions = {}) {
+    super(options);
     this.options = { ...DEFAULT_OPTIONS, ...options };
-    this.protectedPaths = [
-      ...new Set([...protectedLocalPaths(), ...(options.protectedPaths ?? [])]),
-    ];
-    this.includeExtensionSet = new Set(
-      this.options.includeExtensions.map((ext) => ext.toLowerCase())
-    );
-  }
-
-  private isProtected(path: string): boolean {
-    return isProtectedLocalPath(path, this.protectedPaths);
-  }
-
-  private async isProtectedRead(path: string): Promise<boolean> {
-    if (this.isProtected(path)) return true;
-    return isProtectedLocalPath(await realpath(path), this.protectedPaths);
-  }
-
-  /**
-   * The chunkable-file gate, shared by every walker and by the explicit
-   * file-list path so all three agree on what exists. `relPath` is the path
-   * relative to the scan root (OS-native separators accepted).
-   */
-  private isChunkable(absOrRelPath: string, relPath: string): boolean {
-    if (
-      !this.includeExtensionSet.has(extname(absOrRelPath).toLowerCase()) &&
-      !isCursorRulePath(relPath)
-    ) {
-      return false;
-    }
-    if (isAuthorityOnlyPath(relPath)) {
-      logDebug('Skipping chunking for authority-collected path', { relPath });
-      return false;
-    }
-    return true;
-  }
-
-  /**
-   * Read `.gitignore` + `.mnemonikignore` + `.gitattributes` in `absDir` and
-   * compile them into one layer (the two ignore files are unioned -
-   * `.mnemonikignore` is just an additional gitignore-syntax exclusion list).
-   * Returns null when none of the three exists. FAIL-CLOSED: a non-ENOENT read
-   * error (EISDIR, EACCES, timeout) latches `walk.complete = false` so callers
-   * deriving *removals* from the walk treat it as untrustworthy rather than
-   * silently ignoring nothing.
-   */
-  private async loadIgnoreLayer(
-    absDir: string,
-    baseRel: string,
-    walk: WalkState,
-    rootPath: string
-  ): Promise<IgnoreLayer | null> {
-    if (this.isProtected(absDir)) return null;
-    const read = async (name: string): Promise<string[] | null> => {
-      const filePath = join(absDir, name);
-      try {
-        const content = await withTimeout(
-          this.readConfined(filePath, rootPath),
-          FILE_OP_TIMEOUT_MS,
-          `readFile timed out: ${filePath}`
-        );
-        return content.split(/\r?\n/);
-      } catch (err) {
-        if ((err as Error).message === 'symlink_skipped') return null;
-        if ((err as NodeJS.ErrnoException)?.code !== 'ENOENT') {
-          walk.complete = false;
-          logDebug('Rule-file read failed (fail-closed)', { path: filePath, err });
-        }
-        return null;
-      }
-    };
-    const patterns: string[] = [];
-    for (const name of IGNORE_FILE_NAMES) {
-      const lines = await read(name);
-      if (lines) patterns.push(...lines);
-    }
-    const attributeLines = (await read(ATTRIBUTES_FILE_NAME)) ?? [];
-    if (patterns.length > 0 || attributeLines.length > 0) {
-      walk.rules?.set(baseRel, { ignore: patterns, attributes: attributeLines });
-    }
-    const attributes =
-      attributeLines.length > 0 ? parseGitattributes(attributeLines.join('\n')) : [];
-    // `ignore` tolerates blank/comment lines; null when nothing meaningful.
-    if (patterns.length === 0 && attributes.length === 0) return null;
-    return {
-      baseRel,
-      ig: patterns.length > 0 ? ignore().add(patterns) : null,
-      hasNegation: patterns.some((line) => line.trimStart().startsWith('!')),
-      attributes: attributes.length > 0 ? attributes : null,
-    };
-  }
-
-  /**
-   * What the .gitignore / .mnemonikignore layers in `stack` say about
-   * `entryRel` (relative to the scan root). Tested deepest-first: the first
-   * layer that explicitly ignores OR re-includes (via a `!negation`) wins,
-   * matching git's rule that a deeper `.gitignore` overrides a shallower one.
-   * Directories are tested with a trailing slash so `foo/` dir-only rules
-   * match the directory itself. `'unignored'` means a negation matched this
-   * path itself; a re-included parent directory leaves its files `'unmatched'`.
-   */
-  private ignoreVerdict(entryRel: string, isDir: boolean, stack: IgnoreStack): IgnoreFileVerdict {
-    for (let i = stack.length - 1; i >= 0; i--) {
-      const layer = stack[i];
-      if (!layer?.ig) continue;
-      const { baseRel, ig } = layer;
-      let sub: string;
-      if (baseRel === '') {
-        sub = entryRel;
-      } else if (entryRel === baseRel || !entryRel.startsWith(baseRel + '/')) {
-        continue; // this layer does not cover entryRel
-      } else {
-        sub = entryRel.slice(baseRel.length + 1);
-      }
-      if (!sub) continue;
-      const res = ig.test(isDir ? sub + '/' : sub);
-      if (res.ignored) return 'ignored';
-      if (res.unignored) return 'unignored';
-    }
-    return 'unmatched';
-  }
-
-  private ignoredByStack(entryRel: string, isDir: boolean, stack: IgnoreStack): boolean {
-    return this.ignoreVerdict(entryRel, isDir, stack) === 'ignored';
-  }
-
-  private static attributeLayers(stack: IgnoreStack): GitattributesLayer[] {
-    return stack.flatMap((layer) =>
-      layer.attributes ? [{ baseRel: layer.baseRel, rules: layer.attributes }] : []
-    );
-  }
-
-  /**
-   * Skip this directory? User-ignored directories are skipped as before. A
-   * directory a soft default excludes (vendor/, generated/, ...) is pruned too -
-   * nothing under it can be indexed - UNLESS a rule above it could re-include
-   * something inside it: a `!pattern` or a `-linguist-*` attribute. Then the
-   * walk descends and judges each file, so the override works. Rule files
-   * INSIDE a pruned directory are never read: a vendored library's own
-   * .gitignore negations are not the project's intent.
-   */
-  private skipDirectory(relPath: string, stack: IgnoreStack): boolean {
-    const posixRel = relPath.split(sep).join('/');
-    const verdict = this.ignoreVerdict(posixRel, true, stack);
-    if (verdict === 'ignored') return true;
-    if (verdict === 'unignored') return false;
-    if (!softDefaultForDirectory(posixRel)) return false;
-    const canReinclude =
-      stack.some((layer) => layer.hasNegation) ||
-      gitattributesCanReinclude(CodeScanner.attributeLayers(stack));
-    return !canReinclude;
-  }
-
-  /**
-   * The index decision for a file from its path and the rule stack: `null`
-   * when a user ignore rule excludes it, otherwise the classifier's answer
-   * (precedence in indexClassifier.ts). Content rules are the caller's next
-   * step for an included path without an `override`.
-   */
-  private classifyEntry(relPath: string, stack: IgnoreStack): IndexClassification | null {
-    const posixRel = relPath.split(sep).join('/');
-    const verdict = this.ignoreVerdict(posixRel, false, stack);
-    if (verdict === 'ignored') return null;
-    return classifyForIndex(posixRel, {
-      ignoreFileVerdict: verdict,
-      attributes: CodeScanner.attributeLayers(stack),
-    });
-  }
-
-  /**
-   * Reconstruct the root->parent ignore-layer stack for a single file, for the
-   * incremental `scanFiles` path (which has a flat file list, not a recursive
-   * walk). Layers are memoized per directory in `cache` across a batch.
-   */
-  private async buildIgnoreStackForFile(
-    fileRel: string,
-    rootPath: string,
-    cache: Map<string, IgnoreLayer | null>,
-    walk: WalkState
-  ): Promise<IgnoreStack> {
-    const dirs = [''];
-    let acc = '';
-    for (const seg of fileRel.split('/').slice(0, -1)) {
-      acc = acc === '' ? seg : `${acc}/${seg}`;
-      dirs.push(acc);
-    }
-    const stack: IgnoreStack = [];
-    for (const dir of dirs) {
-      let layer = cache.get(dir);
-      if (layer === undefined) {
-        layer = await this.loadIgnoreLayer(
-          dir === '' ? rootPath : join(rootPath, dir),
-          dir,
-          walk,
-          rootPath
-        );
-        cache.set(dir, layer);
-      }
-      if (layer) stack.push(layer);
-    }
-    return stack;
-  }
-
-  /**
-   * A predicate for walkers outside this class (the daemon's file watcher):
-   * true when the walk would descend into `absDir`, by the same rules
-   * `listFiles` applies - user ignore files and soft-default directories,
-   * with their overrides. Rule files are read once per directory for the life
-   * of the returned predicate; take a fresh one after a rule file changes.
-   * Directories outside `rootPath`, and the root itself, always pass.
-   */
-  directoryFilter(rootPath: string): (absDir: string) => Promise<boolean> {
-    const cache = new Map<string, IgnoreLayer | null>();
-    return async (absDir: string): Promise<boolean> => {
-      try {
-        const root = await this.canonicalRoot(rootPath);
-        const rel = relative(root, absDir).split(sep).join('/');
-        if (!rel || rel.startsWith('..')) return true;
-        const stack = await this.buildIgnoreStackForFile(rel, root, cache, { complete: true });
-        return !this.skipDirectory(rel, stack);
-      } catch {
-        // Fail open: watching a directory the walk would skip costs a watch
-        // handle; not watching one it would index loses changes.
-        return true;
-      }
-    };
-  }
-
-  /**
-   * True when any directory strictly between `rootPath` and `filePath`
-   * starts a subtree this project does not index (see isProjectBoundary). Paths that don't
-   * resolve under `rootPath` are not judged here (return false) - the
-   * server's path validation handles escapes.
-   */
-  private async insideNestedGitBoundary(filePath: string, rootPath: string): Promise<boolean> {
-    const rel = relative(rootPath, filePath);
-    if (!rel || rel.startsWith('..')) return false;
-    const segments = rel.split(sep);
-    let current = rootPath;
-    // Walk intermediate directories only (exclude the file itself).
-    for (let i = 0; i < segments.length - 1; i++) {
-      const segment = segments[i];
-      if (segment === undefined) continue;
-      current = join(current, segment);
-      if (await this.isBoundary(current, rootPath)) return true;
-    }
-    return false;
-  }
-
-  /**
-   * Maximum directory depth for recursive scanning
-   * Prevents runaway recursion on deep/symlinked structures
-   */
-  private static readonly MAX_DEPTH = SCANNER_MAX_DEPTH;
-
-  private rootProjectIds = new Map<
-    string,
-    { at: number; projectId: Promise<string | undefined> }
-  >();
-  /**
-   * isProjectBoundary, with the scan root's own project as the enclosing one.
-   * The root's file is read at most once a minute: a walk tests many folders,
-   * and a project relinked since is seen on the next scan.
-   */
-  private async isBoundary(dirPath: string, rootPath: string): Promise<boolean> {
-    let cached = this.rootProjectIds.get(rootPath);
-    if (!cached || Date.now() - cached.at > 60_000) {
-      cached = { at: Date.now(), projectId: rootProjectId(rootPath) };
-      this.rootProjectIds.set(rootPath, cached);
-    }
-    return isProjectBoundary(dirPath, await cached.projectId);
-  }
-
-  private canonicalRoots = new Map<string, string>();
-  private async canonicalRoot(root: string): Promise<string> {
-    let canonical = this.canonicalRoots.get(root);
-    if (!canonical) {
-      canonical = await realpath(root);
-      this.canonicalRoots.set(root, canonical);
-    }
-    return canonical;
-  }
-
-  static containsCanonical(
-    candidate: string,
-    root: string,
-    paths = { sep, resolve },
-    caseInsensitive = process.platform === 'win32'
-  ): boolean {
-    // realpath expands filesystem aliases (including Windows 8.3 names). Never guess a short name.
-    candidate = paths.resolve(candidate);
-    root = paths.resolve(root);
-    if (caseInsensitive) {
-      candidate = candidate.toLowerCase();
-      root = root.toLowerCase();
-    }
-    return (
-      candidate === root || candidate.startsWith(root.endsWith(paths.sep) ? root : root + paths.sep)
-    );
-  }
-
-  private async confineSymlink(fullPath: string, canonicalRoot: string): Promise<boolean> {
-    // Check each component: explicit-file callers must not bypass the walkers'
-    // no-symlink policy by supplying a path through an in-root directory link.
-    let path = resolve(fullPath);
-    if (!CodeScanner.containsCanonical(path, canonicalRoot)) return false;
-    for (;;) {
-      if (this.skipSymlink(await lstat(path))) throw new Error('symlink_skipped');
-      if (CodeScanner.containsCanonical(canonicalRoot, path)) break;
-      path = dirname(path);
-    }
-    return CodeScanner.containsCanonical(await realpath(fullPath), canonicalRoot);
-  }
-
-  async readConfined(fullPath: string, canonicalRoot: string, enumerated?: Stats): Promise<string> {
-    const expected = enumerated ?? (await lstat(fullPath));
-    if (this.skipSymlink(expected)) throw new Error('symlink_skipped');
-    if (!expected.isFile()) throw new Error('scanner_not_regular');
-    if (!(await this.confineSymlink(fullPath, canonicalRoot)))
-      throw new Error('scanner_containment');
-    const canonical = await realpath(fullPath);
-    if (
-      !CodeScanner.containsCanonical(canonical, canonicalRoot) ||
-      (await this.isProtectedRead(canonical))
-    )
-      throw new Error('scanner_containment');
-    // O_NONBLOCK also prevents a substituted FIFO from hanging before fstat.
-    const flags =
-      constants.O_RDONLY |
-      (constants.O_NONBLOCK ?? 0) |
-      (process.platform === 'win32' ? 0 : (constants.O_NOFOLLOW ?? 0));
-    const handle = await open(fullPath, flags).catch((error: NodeJS.ErrnoException) => {
-      if (error.code === 'ELOOP') {
-        this.skipReasons.symlink_skipped++;
-        throw new Error('symlink_skipped');
-      }
-      throw error;
-    });
-    try {
-      const actual = await handle.stat();
-      if (!actual.isFile()) throw new Error('scanner_not_regular');
-      if (actual.dev !== expected.dev || actual.ino !== expected.ino)
-        throw new Error('scanner_identity_changed');
-      // Windows has no O_NOFOLLOW; lstat is Node's junction/reparse-link check.
-      if (process.platform === 'win32' && this.skipSymlink(await lstat(fullPath)))
-        throw new Error('symlink_skipped');
-      // Linux also proves the opened object is still at the checked path.
-      if (
-        process.platform === 'linux' &&
-        (await realpath(`/proc/self/fd/${handle.fd}`)) !== canonical
-      )
-        throw new Error('scanner_containment');
-      return await handle.readFile('utf-8');
-    } finally {
-      await handle.close();
-    }
   }
 
   /**
    * Scan a directory recursively and extract code chunks
-   * Added max depth (10) to prevent infinite recursion
+   * No depth limit: symlinks are skipped and WalkState.ancestors ends any cycle.
    */
   async scanDirectory(rootPath: string): Promise<CodeChunk[]> {
     const { chunks } = await this.scanDirectoryWithStatus(rootPath);
@@ -1289,144 +255,6 @@ export class CodeScanner {
     return { chunks: await this.scrubChunks(chunks), complete: walk.complete, chunkless, excluded };
   }
 
-  /**
-   * Enumerate scan-eligible relative file paths under `rootPath` without
-   * reading or chunking content. Same walk + ignorePatterns + extension
-   * filter as `scanDirectory`, but bounded to O(file count) directory ops
-   * - cheap enough to call on every periodic reconcile tick.
-   *
-   * Returns relative paths normalized against `rootPath`, matching the
-   * shape the server stores in `memories.metadata->>'filePath'`. Use this
-   * for the scanner reconciliation channel (`POST /api/v1/scan/reconcile`).
-   *
-   * Defensive: skips any path whose `relative()` result contains a `..`
-   * traversal segment (can happen when a symlink resolves under root but
-   * the readdir entry path doesn't normalize cleanly). The server's
-   * `safeScanPath` rejects such paths; filtering here keeps a single
-   * malformed entry from failing the entire reconcile push.
-   */
-  async listFiles(rootPath: string): Promise<string[]> {
-    const { paths } = await this.listFilesWithStatus(rootPath);
-    return paths;
-  }
-
-  /**
-   * Completeness-aware variant of `listFiles`. `complete === false` means
-   * an fs error truncated part of the walk - the returned paths may be
-   * missing files that still exist on disk. The daemon's reconcile channel
-   * MUST NOT derive removedFiles from an incomplete inventory (a truncated
-   * walk otherwise reads as mass deletion and deprecates live memories).
-   */
-  async listFilesWithStatus(rootPath: string): Promise<{
-    paths: string[];
-    complete: boolean;
-    /**
-     * Listed paths a project rule re-included (a `!pattern` or a
-     * `-linguist-*` attribute). Content rules do not apply to them; every
-     * other listed path still answers to the content rules when read.
-     */
-    overridden: Set<string>;
-  }> {
-    rootPath = await this.canonicalRoot(rootPath);
-    const paths: string[] = [];
-    const overridden = new Set<string>();
-    const walk: WalkState = { complete: true, rules: new Map() };
-    await this.traversePaths(rootPath, rootPath, paths, 0, walk, [], overridden);
-    // Only a complete walk replaces the record: a partial one may be missing
-    // rule files, and uploading it would tell the server fewer rules exist.
-    if (walk.complete && walk.rules) this.ruleFilesByRoot.set(rootPath, walk.rules);
-    return {
-      paths: paths.filter((p) => !/(^|[/\\])\.\.([/\\]|$)/.test(p)),
-      complete: walk.complete,
-      overridden,
-    };
-  }
-
-  /**
-   * The rule files the last complete `listFilesWithStatus` walk of `rootPath`
-   * applied, by directory ('' = root). Null before the first complete walk.
-   * The daemon uploads these lines so the server judges paths as it did.
-   */
-  async indexRuleFiles(rootPath: string): Promise<ReadonlyMap<string, IndexRuleFiles> | null> {
-    return this.ruleFilesByRoot.get(await this.canonicalRoot(rootPath)) ?? null;
-  }
-
-  private readonly ruleFilesByRoot = new Map<string, Map<string, IndexRuleFiles>>();
-
-  private async traversePaths(
-    currentPath: string,
-    rootPath: string,
-    out: string[],
-    depth: number,
-    walk: WalkState,
-    stack: IgnoreStack = [],
-    overridden?: Set<string>
-  ): Promise<void> {
-    if (depth >= CodeScanner.MAX_DEPTH) return;
-    if (this.isProtected(currentPath)) return;
-    let entries: string[];
-    try {
-      if (!(await this.confineSymlink(currentPath, rootPath)))
-        throw new Error('scanner_containment');
-      entries = await withTimeout(
-        readdir(currentPath),
-        FILE_OP_TIMEOUT_MS,
-        `readdir timed out: ${currentPath}`
-      );
-    } catch (error) {
-      walk.complete = false;
-      logDebug('Error traversing directory (listFiles)', { path: currentPath, error });
-      return;
-    }
-    const dirRel =
-      currentPath === rootPath ? '' : relative(rootPath, currentPath).split(sep).join('/');
-    const localLayer = await this.loadIgnoreLayer(currentPath, dirRel, walk, rootPath);
-    const localStack = localLayer ? [...stack, localLayer] : stack;
-    for (const entry of entries) {
-      const fullPath = join(currentPath, entry);
-      const relativePath = relative(rootPath, fullPath);
-      if (this.isProtected(fullPath)) continue;
-      if (this.shouldIgnore(relativePath)) continue;
-
-      // Per-entry isolation: a single bad entry (dangling symlink, raced
-      // deletion, permission denial, fs timeout) must not abort the rest of
-      // the directory - but the inventory is no longer removal-trustworthy.
-      try {
-        const lstats = await withTimeout(
-          lstat(fullPath),
-          FILE_OP_TIMEOUT_MS,
-          `lstat timed out: ${fullPath}`
-        );
-        if (this.skipSymlink(lstats)) continue;
-        const stats = lstats;
-
-        if (stats.isDirectory()) {
-          // .gitignore/.mnemonikignore, then soft-default directories
-          if (this.skipDirectory(relativePath, localStack)) continue;
-          if (await this.isBoundary(fullPath, rootPath)) continue; // a subproject or worktree
-          await this.traversePaths(
-            fullPath,
-            rootPath,
-            out,
-            depth + 1,
-            walk,
-            localStack,
-            overridden
-          );
-        } else if (stats.isFile()) {
-          if (!this.isChunkable(fullPath, relativePath)) continue;
-          const decision = this.classifyEntry(relativePath, localStack);
-          if (!decision?.include) continue;
-          out.push(relativePath);
-          if (decision.override) overridden?.add(relativePath);
-        }
-      } catch (error) {
-        walk.complete = false;
-        logDebug('Error traversing directory (listFiles)', { path: fullPath, error });
-      }
-    }
-  }
-
   /**
    * Scan specific files and extract code chunks.
    * Pass rootPath to compute proper relative file paths in chunk metadata.
@@ -1568,11 +396,6 @@ export class CodeScanner {
     chunkless?: string[],
     excluded?: IndexExcludedFile[]
   ): Promise<void> {
-    // Prevent infinite recursion
-    if (depth >= CodeScanner.MAX_DEPTH) {
-      logDebug('Max directory depth reached, skipping', { path: currentPath, depth });
-      return;
-    }
     if (this.isProtected(currentPath)) return;
 
     // Wrap readdir with timeout to prevent hanging
@@ -1580,13 +403,11 @@ export class CodeScanner {
     try {
       if (!(await this.confineSymlink(currentPath, rootPath)))
         throw new Error('scanner_containment');
-      entries = await withTimeout(
-        readdir(currentPath),
-        FILE_OP_TIMEOUT_MS,
-        `readdir timed out: ${currentPath}`
-      );
+      // The root is the first ancestor; every other directory enters before it recurses.
+      if (depth === 0 && !(await enterDirectory(walk, currentPath))) return;
+      entries = await fileOp(() => readdir(currentPath), `readdir: ${currentPath}`);
     } catch (error) {
-      walk.complete = false;
+      markUnreadable(walk, currentPath, error);
       logDebug('Error traversing directory', { path: currentPath, error });
       return;
     }
@@ -1611,11 +432,7 @@ export class CodeScanner {
       // deletion, permission denial, fs timeout) must not abort the rest of
       // the directory - but the scan is no longer removal-trustworthy.
       try {
-        const lstats = await withTimeout(
-          lstat(fullPath),
-          FILE_OP_TIMEOUT_MS,
-          `lstat timed out: ${fullPath}`
-        );
+        const lstats = await fileOp(() => lstat(fullPath), `lstat: ${fullPath}`);
 
         if (this.skipSymlink(lstats)) continue;
         const stats = lstats;
@@ -1624,16 +441,22 @@ export class CodeScanner {
           // .gitignore/.mnemonikignore, then soft-default directories
           if (this.skipDirectory(relativePath, localStack)) continue;
           if (await this.isBoundary(fullPath, rootPath)) continue; // a subproject or worktree
-          await this.traverseDirectory(
-            fullPath,
-            rootPath,
-            chunks,
-            depth + 1,
-            walk,
-            localStack,
-            chunkless,
-            excluded
-          );
+          const leave = await enterDirectory(walk, fullPath);
+          if (!leave) continue;
+          try {
+            await this.traverseDirectory(
+              fullPath,
+              rootPath,
+              chunks,
+              depth + 1,
+              walk,
+              localStack,
+              chunkless,
+              excluded
+            );
+          } finally {
+            leave();
+          }
         } else if (stats.isFile()) {
           if (!this.isChunkable(fullPath, relativePath)) continue;
           const decision = this.classifyEntry(relativePath, localStack);
@@ -1645,49 +468,12 @@ export class CodeScanner {
           chunks.push(...fileChunks);
         }
       } catch (error) {
-        walk.complete = false;
+        markUnreadable(walk, fullPath, error);
         logDebug('Error traversing directory', { path: fullPath, error });
       }
     }
   }
 
-  /**
-   * Check if path should be ignored
-   * Fixed glob-to-regex conversion and substring matching.
-   * - Escape regex special chars before replacing * with .*
-   * - Replace ALL * occurrences (not just the first)
-   * - For non-glob patterns, match on path segments to avoid false positives
-   *   (e.g., '.env' should not match '.environment.ts')
-   */
-  private shouldIgnore(path: string): boolean {
-    // Credential-bearing file kinds first, and outside the overridable
-    // `ignorePatterns` list - a caller-supplied pattern set must not be able
-    // to opt a project back into indexing its own private keys.
-    if (isSecretFile(path)) return true;
-    const segments = path.split('/');
-    return this.options.ignorePatterns.some((pattern) => {
-      if (pattern.includes('*')) {
-        // A leading '/' opts a glob pattern into SEGMENT-ANCHORED matching:
-        // the pattern body must start at the beginning of the path or right
-        // after a '/'. Without it, glob patterns keep their historical
-        // unanchored substring semantics - anchoring is opt-in per pattern so
-        // this change cannot alter any existing pattern's behavior.
-        // (Motivating bug: unanchored 'tests/fixtures/*' also matched
-        // 'integration-tests/fixtures/...' in every indexed project.)
-        const anchored = pattern.startsWith('/');
-        const body = anchored ? pattern.slice(1) : pattern;
-        // Escape regex special chars, then replace all * with .*
-        const escaped = body.replace(/[.+?^${}()|[\]\\]/g, '\\$&');
-        const source = escaped.replace(/\*/g, '.*');
-        const regex = new RegExp(anchored ? `(^|/)${source}` : source);
-        return regex.test(path);
-      }
-      // For non-glob patterns, check if any path segment matches exactly
-      // or if the full path ends with the pattern (for extension-like patterns)
-      return segments.some((segment) => segment === pattern) || path.endsWith('/' + pattern);
-    });
-  }
-
   /**
    * Parse a file and extract code chunks.
    * Size limit is the shared `MAX_SCANNED_FILE_BYTES` ceiling - the same
@@ -1708,9 +494,7 @@ export class CodeScanner {
     try {
       // Check file size before reading to avoid memory issues
       // Wrap stat with timeout
-      const stats =
-        enumerated ??
-        (await withTimeout(lstat(filePath), FILE_OP_TIMEOUT_MS, `stat timed out: ${filePath}`));
+      const stats = enumerated ?? (await fileOp(() => lstat(filePath), `stat: ${filePath}`));
       if (stats.size > CodeScanner.MAX_FILE_SIZE) {
         logDebug('Skipping file exceeding size limit', {
           filePath,
@@ -1721,10 +505,9 @@ export class CodeScanner {
       }
 
       // Wrap readFile with timeout
-      let content = await withTimeout(
-        this.readConfined(filePath, rootPath, stats),
-        FILE_OP_TIMEOUT_MS,
-        `readFile timed out: ${filePath}`
+      let content = await fileOp(
+        () => this.readConfined(filePath, rootPath, stats),
+        `readFile: ${filePath}`
       );
       const relativePath = relative(rootPath, filePath);
       // Content soft defaults (binary, generated header, minified, data dump),
@@ -2510,21 +1293,24 @@ export class CodeScanner {
   ): Promise<{
     files: Array<{ path: string; content: string; hash: string }>;
     complete: boolean;
+    /** Entries a filesystem error or timeout left out. */
+    unreadable: number;
+    /** The first few unreadable entries, with the reason. */
+    unreadableSamples: Array<{ path: string; reason: string }>;
   }> {
     projectRoot = await this.canonicalRoot(projectRoot);
     const out: Array<{ path: string; content: string; hash: string }> = [];
     const walkState: WalkState = { complete: true };
 
-    // M2: cap walk depth to match the scanner's MAX_DEPTH (10)
     const walk = async (dir: string, depth: number, stack: IgnoreStack = []): Promise<void> => {
-      if (depth >= CodeScanner.MAX_DEPTH) return;
       if (this.isProtected(dir)) return;
       let entries;
       try {
+        if (depth === 0 && !(await enterDirectory(walkState, dir))) return;
         if (!(await this.confineSymlink(dir, projectRoot))) throw new Error('scanner_containment');
-        entries = await readdir(dir, { withFileTypes: true });
-      } catch {
-        walkState.complete = false;
+        entries = await fileOp(() => readdir(dir, { withFileTypes: true }), `readdir: ${dir}`);
+      } catch (error) {
+        markUnreadable(walkState, dir, error);
         return;
       }
       const dirRel = dir === projectRoot ? '' : relative(projectRoot, dir).split(sep).join('/');
@@ -2538,9 +1324,9 @@ export class CodeScanner {
 
         let stats: Stats;
         try {
-          stats = await lstat(full);
-        } catch {
-          walkState.complete = false;
+          stats = await fileOp(() => lstat(full), `lstat: ${full}`);
+        } catch (error) {
+          markUnreadable(walkState, full, error);
           continue;
         }
         if (this.skipSymlink(stats)) continue;
@@ -2550,12 +1336,21 @@ export class CodeScanner {
         if (this.ignoredByStack(rel, isDir, localStack)) continue; // .gitignore/.mnemonikignore
         if (isDir) {
           if (await this.isBoundary(full, projectRoot)) continue; // a subproject or worktree
-          await walk(full, depth + 1, localStack);
+          const leave = await enterDirectory(walkState, full);
+          if (!leave) continue;
+          try {
+            await walk(full, depth + 1, localStack);
+          } finally {
+            leave();
+          }
         } else if (isFile && AUTHORITY_FILE_MATCHERS.some((m) => m(rel))) {
           let file: { path: string; content: string; hash: string } | undefined;
           try {
             if (stats.size > MAX_SCANNED_FILE_BYTES) continue;
-            let content = await this.readConfined(full, projectRoot, stats);
+            let content = await fileOp(
+              () => this.readConfined(full, projectRoot, stats),
+              `readFile: ${full}`
+            );
             // Authority files honor the same single ceiling as chunked files
             // (MAX_SCANNED_FILE_BYTES). Manifests are tiny; one this large is
             // anomalous, so say so rather than dropping it silently - a
@@ -2578,9 +1373,9 @@ export class CodeScanner {
             }
             const hash = createHash('sha256').update(content).digest('hex');
             file = { path: rel, content, hash };
-          } catch {
+          } catch (error) {
             /* unreadable - skip */
-            walkState.complete = false;
+            markUnreadable(walkState, full, error);
           }
           // The consumer owns backpressure; its abort must leave the walk immediately.
           if (file) {
@@ -2592,6 +1387,11 @@ export class CodeScanner {
     };
 
     await walk(projectRoot, 0);
-    return { files: out, complete: walkState.complete };
+    return {
+      files: out,
+      complete: walkState.complete,
+      unreadable: walkState.unreadable ?? 0,
+      unreadableSamples: walkState.unreadableSamples ?? [],
+    };
   }
 }