@maxgfr/codeindex 2.12.0 → 2.14.0

package/src/walk.ts

@@ -1,15 +1,18 @@
-import { readdirSync, statSync, lstatSync, readFileSync, realpathSync } from "node:fs";
+import { readdirSync, statSync, lstatSync, readFileSync, realpathSync, type Dirent } from "node:fs";
 import { join, relative, sep, extname } from "node:path";
 import { parseGitignore, isIgnored, type IgnoreRule } from "./ignore.js";
 
 // Directories that never carry signal for a documentation/code question and
 // would bloat the index (dependencies, build output, VCS internals, caches).
+// .codeindex is the engine's OWN output (index artifacts, pulled models, MCP
+// memories) — indexing it would feed memories into search and churn the scan
+// fingerprint on every write_memory (issue #12).
 // Exported so grep.ts can align ripgrep's universe with the walker's.
 export const IGNORE_DIRS = new Set([
   ".git", "node_modules", ".pnpm", "bower_components", "vendor", "dist", "build", "out",
   "target", ".next", ".nuxt", ".svelte-kit", ".turbo", "coverage", "__pycache__", ".venv",
   "venv", ".tox", ".mypy_cache", ".pytest_cache", ".gradle", ".idea", ".vscode", ".cache",
-  "tmp", ".ultraindex", "Pods", "DerivedData", ".terraform", "elm-stuff", ".dart_tool",
+  "tmp", ".ultraindex", ".codeindex", "Pods", "DerivedData", ".terraform", "elm-stuff", ".dart_tool",
 ]);
 
 // Lockfiles: huge, machine-generated, and pure noise for a code/docs question —
@@ -36,6 +39,13 @@ export interface WalkOptions {
   // semantics — see ignore.ts). Default TRUE: an ignored file is noise for
   // every consumer; pass false to index generated/ignored trees deliberately.
   gitignore?: boolean;
+  // Directory names to skip, REPLACING the default set entirely (not merging
+  // with it). IGNORE_DIRS is a public export, so consumers compose
+  // `[...IGNORE_DIRS, "extra"]` — or filter it — themselves; replace is the
+  // simplest contract. Deliberate scope boundary: grep.ts (the ripgrep
+  // universe) and the MCP server keep the DEFAULT set — recall consumers
+  // (e.g. ultrasec) consume scan/extract, not grep.
+  ignoreDirs?: string[];
 }
 
 export interface WalkedFile {
@@ -65,6 +75,9 @@ export function walk(root: string, opts: WalkOptions = {}): WalkResult {
   const maxFileBytes = opts.maxFileBytes ?? 1024 * 1024;
   const maxFiles = opts.maxFiles ?? DEFAULT_MAX_FILES;
   const useGitignore = opts.gitignore !== false;
+  // Effective ignored-directory set, built once: the caller's replacement when
+  // given (see WalkOptions.ignoreDirs — replace, never merge), else the default.
+  const ignoreDirs = opts.ignoreDirs ? new Set(opts.ignoreDirs) : IGNORE_DIRS;
   const out: WalkedFile[] = [];
   let capped = false;
   let excluded = 0;
@@ -100,32 +113,49 @@ export function walk(root: string, opts: WalkOptions = {}): WalkResult {
     if (seenDirs.has(real)) continue;
     seenDirs.add(real);
     if (!contained(real)) continue; // dir symlink escaping the repo
-    let entries: string[];
+    let entries: Dirent[];
     try {
       // Sorted so the walk order — and therefore WHICH files survive a
-      // maxFiles cap — is identical across filesystems and machines.
-      entries = readdirSync(frame.dir).sort();
+      // maxFiles cap — is identical across filesystems and machines. Dirents
+      // sort by .name under the same code-unit comparison the bare-string
+      // sort used, so the order is byte-identical to the previous readdir.
+      entries = readdirSync(frame.dir, { withFileTypes: true }).sort((a, b) =>
+        a.name < b.name ? -1 : a.name > b.name ? 1 : 0,
+      );
     } catch {
       continue;
     }
     let rules = frame.rules;
-    if (useGitignore && entries.includes(".gitignore")) {
+    if (useGitignore && entries.some((e) => e.name === ".gitignore")) {
       const parsed = parseGitignore(readText(join(frame.dir, ".gitignore")), frame.rel);
       if (parsed.length) rules = [...rules, ...parsed];
     }
-    for (const name of entries) {
+    for (const entry of entries) {
+      const name = entry.name;
       const abs = join(frame.dir, name);
       const rel = frame.rel ? `${frame.rel}/${name}` : name;
+      // The dirent type IS the lstat type (Node lstats internally when the
+      // filesystem can't supply it), so no lstat is needed to detect links.
+      const isLink = entry.isSymbolicLink();
+      // Ignored-directory boundary (node_modules, .git…): skip on the dirent
+      // type alone — ZERO stat syscalls. A symlink reports isDirectory()
+      // false on its dirent and falls through to the stat-based
+      // classification below, so a link named node_modules still classifies
+      // by its target exactly as before.
+      if (entry.isDirectory() && ignoreDirs.has(name)) continue;
       let st;
-      let isLink: boolean;
       try {
-        st = statSync(abs);
-        isLink = lstatSync(abs).isSymbolicLink();
+        // Non-links: a single lstatSync supplies isDirectory/isFile/size/
+        // mtimeMs — field-for-field what the previous statSync returned,
+        // since with no link to follow the two calls are identical. Links:
+        // keep the following statSync so the entry classifies by its TARGET;
+        // a broken link throws here and is skipped, same as before.
+        st = isLink ? statSync(abs) : lstatSync(abs);
       } catch {
         continue;
       }
       if (st.isDirectory()) {
-        if (IGNORE_DIRS.has(name)) continue;
+        if (ignoreDirs.has(name)) continue;
         // An in-repo DIRECTORY symlink is skipped entirely: its target is (or
         // will be) walked under its canonical name, and letting both paths race
         // through the cycle guard would keep whichever readdir served first —