codesift-mcp 0.9.5 → 0.9.10

package/dist/utils/walk.js

@@ -1,156 +1,28 @@
-import { readdir, stat } from "node:fs/promises";
-import { join, relative, extname } from "node:path";
-import picomatch from "picomatch";
-/**
- * Directories to skip during filesystem walks.
- * Shared by index-tools, search-tools, and (via toIgnorePatterns) watcher.
- */
-export const IGNORE_DIRS = new Set([
-    "node_modules", ".git", "dist", "build", "coverage",
-    ".codesift", ".next", "__pycache__", ".pytest_cache",
-    ".venv", "venv", ".tox", ".mypy_cache", ".turbo",
-    "generated", "audit-results", ".backup", "jscpd-report",
-    "helpscout_export", ".output",
-    ".pnpm", // pnpm symlink forest — recursing here can blow up file count
-]);
-/**
- * Convert IGNORE_DIRS to chokidar-compatible glob patterns.
- * e.g., "node_modules" -> "**\/node_modules/**"
- */
-export function toIgnorePatterns() {
-    return [...IGNORE_DIRS].map((dir) => `**/${dir}/**`);
-}
-const DEFAULT_MAX_FILE_SIZE = 1_000_000; // 1MB
-/**
- * Backup and editor-generated files to exclude by default.
- * These patterns commonly appear in working directories and cause:
- * - Index duplication (e.g. Mobi2 had Survey.php + Survey copy.php)
- * - False references pointing at backup files
- * - Wasted parser time on abandoned code
- *
- * Disabled with env var CODESIFT_INCLUDE_BACKUPS=1.
- */
-export const BACKUP_FILE_PATTERNS = [
-    /copy\.php$/i, // macOS Finder "Duplicate" output
-    /\.bak$/i, // generic backup
-    /\.orig$/i, // merge conflict leftover
-    /~$/, // emacs/joe backup
-    /\.swp$/i, // vim swap
-    /\.swo$/i, // vim swap
-    /\.DS_Store$/, // macOS finder metadata
-];
-/**
- * Walk a directory tree collecting files.
- * Skips directories listed in IGNORE_DIRS and hidden directories (dot-prefixed).
- */
+import { relative } from "node:path";
+import { createFileFilter, IGNORE_DIRS, toIgnorePatterns, BACKUP_FILE_PATTERNS } from "./walk/filters.js";
+import { WalkLimits } from "./walk/limits.js";
+import { readStats } from "./walk/errors.js";
+import { traverseDirectory } from "./walk/traversal.js";
+export { BACKUP_FILE_PATTERNS, IGNORE_DIRS, toIgnorePatterns };
 export async function walkDirectory(rootPath, options) {
+    const normalizedOptions = options ?? {};
     const files = [];
-    const maxFileSize = options?.maxFileSize ?? DEFAULT_MAX_FILE_SIZE;
-    const maxFiles = options?.maxFiles ?? Infinity;
-    const fileFilter = options?.fileFilter;
-    const includePaths = options?.includePaths;
-    const useRelative = options?.relative ?? false;
-    const followSymlinks = options?.followSymlinks ?? false;
-    const visitedInodes = new Set();
-    let limitReached = false;
-    // Compile exclude patterns once (picomatch)
-    let isExcluded = null;
-    if (options?.excludePatterns && options.excludePatterns.length > 0) {
-        try {
-            isExcluded = picomatch(options.excludePatterns, { dot: true });
-        }
-        catch {
-            // Malformed patterns — warn and skip filtering
-            console.warn("[codesift] walkDirectory: invalid excludePatterns, ignoring");
-        }
-    }
-    async function walk(dirPath) {
-        if (limitReached)
-            return;
-        let entries;
-        try {
-            entries = await readdir(dirPath, { withFileTypes: true });
-        }
-        catch {
-            return; // permission denied, etc.
-        }
-        for (const entry of entries) {
-            if (limitReached)
-                return;
-            const fullPath = join(dirPath, entry.name);
-            const isSymlink = entry.isSymbolicLink();
-            let isDir = entry.isDirectory();
-            let isFile = entry.isFile();
-            // Resolve symlinks when followSymlinks is enabled
-            if (isSymlink && followSymlinks) {
-                try {
-                    const resolved = await stat(fullPath);
-                    const ino = resolved.ino;
-                    if (visitedInodes.has(ino)) {
-                        // Cycle detected — skip
-                        continue;
-                    }
-                    visitedInodes.add(ino);
-                    isDir = resolved.isDirectory();
-                    isFile = resolved.isFile();
-                }
-                catch {
-                    // Broken symlink — skip silently
-                    continue;
-                }
-            }
-            else if (isSymlink) {
-                // Not following symlinks — skip
-                continue;
-            }
-            if (isDir) {
-                if (IGNORE_DIRS.has(entry.name) || entry.name.startsWith(".")) {
-                    continue;
-                }
-                await walk(fullPath);
-            }
-            else if (isFile) {
-                const ext = extname(entry.name);
-                // Exclude backup / editor-generated files unless env var opts out.
-                // Checked before file filter so the noise never reaches the caller.
-                if (process.env.CODESIFT_INCLUDE_BACKUPS !== "1" &&
-                    BACKUP_FILE_PATTERNS.some((re) => re.test(entry.name)))
-                    continue;
-                // Apply caller's file filter (e.g. language check, binary exclusion)
-                if (fileFilter && !fileFilter(ext, entry.name))
-                    continue;
-                // Filter by include paths if specified
-                if (includePaths && includePaths.length > 0) {
-                    const relPath = relative(rootPath, fullPath);
-                    const matches = includePaths.some((p) => relPath.startsWith(p));
-                    if (!matches)
-                        continue;
-                }
-                // Apply exclude patterns
-                if (isExcluded) {
-                    const relPath = relative(rootPath, fullPath);
-                    if (isExcluded(relPath))
-                        continue;
-                }
-                // Skip files that are too large
-                try {
-                    const fileStat = await stat(fullPath);
-                    if (fileStat.size > maxFileSize)
-                        continue;
-                }
-                catch {
-                    continue;
-                }
-                files.push(useRelative ? relative(rootPath, fullPath) : fullPath);
-                if (files.length >= maxFiles) {
-                    console.warn(`[codesift] walkDirectory: reached ${maxFiles} file limit, returning partial results`);
-                    limitReached = true;
-                    return;
-                }
-            }
-        }
-    }
-    await walk(rootPath);
+    const limits = new WalkLimits(normalizedOptions.maxFileSize, normalizedOptions.maxFiles);
+    const filters = createFileFilter(rootPath, normalizedOptions);
+    await traverseDirectory(rootPath, {
+        followSymlinks: normalizedOptions.followSymlinks ?? false,
+        shouldSkipDirectory: filters.shouldSkipDirectory,
+        onFile: async ({ fullPath, entry, stats }) => {
+            if (!filters.shouldIncludeFile(fullPath, entry.name))
+                return true;
+            const fileStats = stats ?? (await readStats(fullPath));
+            if (!fileStats || fileStats.size > limits.maxFileSize)
+                return true;
+            files.push(normalizedOptions.relative ? relative(rootPath, fullPath) : fullPath);
+            limits.acceptFile();
+            return limits.canContinue;
+        },
+    });
     return files;
 }
 //# sourceMappingURL=walk.js.map

package/dist/utils/walk.js.map

@@ -1 +1 @@
-{"version":3,"file":"walk.js","sourceRoot":"","sources":["../../src/utils/walk.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,kBAAkB,CAAC;AACjD,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AACpD,OAAO,SAAS,MAAM,WAAW,CAAC;AAElC;;;GAGG;AACH,MAAM,CAAC,MAAM,WAAW,GAAG,IAAI,GAAG,CAAC;IACjC,cAAc,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,UAAU;IACnD,WAAW,EAAE,OAAO,EAAE,aAAa,EAAE,eAAe;IACpD,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,aAAa,EAAE,QAAQ;IAChD,WAAW,EAAE,eAAe,EAAE,SAAS,EAAE,cAAc;IACvD,kBAAkB,EAAE,SAAS;IAC7B,OAAO,EAAE,8DAA8D;CACxE,CAAC,CAAC;AAEH;;;GAGG;AACH,MAAM,UAAU,gBAAgB;IAC9B,OAAO,CAAC,GAAG,WAAW,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,MAAM,GAAG,KAAK,CAAC,CAAC;AACvD,CAAC;AAED,MAAM,qBAAqB,GAAG,SAAS,CAAC,CAAC,MAAM;AAE/C;;;;;;;;GAQG;AACH,MAAM,CAAC,MAAM,oBAAoB,GAAa;IAC5C,aAAa,EAAQ,kCAAkC;IACvD,SAAS,EAAY,iBAAiB;IACtC,UAAU,EAAW,0BAA0B;IAC/C,IAAI,EAAiB,mBAAmB;IACxC,SAAS,EAAY,WAAW;IAChC,SAAS,EAAY,WAAW;IAChC,aAAa,EAAQ,wBAAwB;CAC9C,CAAC;AAkDF;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,aAAa,CACjC,QAAgB,EAChB,OAAqB;IAErB,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,MAAM,WAAW,GAAG,OAAO,EAAE,WAAW,IAAI,qBAAqB,CAAC;IAClE,MAAM,QAAQ,GAAG,OAAO,EAAE,QAAQ,IAAI,QAAQ,CAAC;IAC/C,MAAM,UAAU,GAAG,OAAO,EAAE,UAAU,CAAC;IACvC,MAAM,YAAY,GAAG,OAAO,EAAE,YAAY,CAAC;IAC3C,MAAM,WAAW,GAAG,OAAO,EAAE,QAAQ,IAAI,KAAK,CAAC;IAC/C,MAAM,cAAc,GAAG,OAAO,EAAE,cAAc,IAAI,KAAK,CAAC;IACxD,MAAM,aAAa,GAAG,IAAI,GAAG,EAAU,CAAC;IACxC,IAAI,YAAY,GAAG,KAAK,CAAC;IAEzB,4CAA4C;IAC5C,IAAI,UAAU,GAAuC,IAAI,CAAC;IAC1D,IAAI,OAAO,EAAE,eAAe,IAAI,OAAO,CAAC,eAAe,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACnE,IAAI,CAAC;YACH,UAAU,GAAG,SAAS,CAAC,OAAO,CAAC,eAAe,EAAE,EAAE,GAAG,EAAE,IAAI,EAAE,CAAC,CAAC;QACjE,CAAC;QAAC,MAAM,CAAC;YACP,+CAA+C;YAC/C,OAAO,CAAC,IAAI,CAAC,6DAA6D,CAAC,CAAC;QAC9E,CAAC;IACH,CAAC;IAED,KAAK,UAAU,IAAI,CAAC,OAAe;QACjC,IAAI,YAAY;YAAE,OAAO;QAEzB,IAAI,OAAO,CAAC;QACZ,IAAI,CAAC;YACH,OAAO,GAAG,MAAM,OAAO,CAAC,OAAO,EAAE,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC,CAAC;QAC5D,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,CAAC,0BAA0B;QACpC,CAAC;QAED,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;YAC5B,IAAI,YAAY;gBAAE,OAAO;YACzB,MAAM,QAAQ,GAAG,IAAI,CAAC,OAAO,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC;YAE3C,MAAM,SAAS,GAAG,KAAK,CAAC,cAAc,EAAE,CAAC;YACzC,IAAI,KAAK,GAAG,KAAK,CAAC,WAAW,EAAE,CAAC;YAChC,IAAI,MAAM,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC;YAE5B,kDAAkD;YAClD,IAAI,SAAS,IAAI,cAAc,EAAE,CAAC;gBAChC,IAAI,CAAC;oBACH,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,CAAC;oBACtC,MAAM,GAAG,GAAG,QAAQ,CAAC,GAAG,CAAC;oBACzB,IAAI,aAAa,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC;wBAC3B,wBAAwB;wBACxB,SAAS;oBACX,CAAC;oBACD,aAAa,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;oBACvB,KAAK,GAAG,QAAQ,CAAC,WAAW,EAAE,CAAC;oBAC/B,MAAM,GAAG,QAAQ,CAAC,MAAM,EAAE,CAAC;gBAC7B,CAAC;gBAAC,MAAM,CAAC;oBACP,iCAAiC;oBACjC,SAAS;gBACX,CAAC;YACH,CAAC;iBAAM,IAAI,SAAS,EAAE,CAAC;gBACrB,gCAAgC;gBAChC,SAAS;YACX,CAAC;YAED,IAAI,KAAK,EAAE,CAAC;gBACV,IAAI,WAAW,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,KAAK,CAAC,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC;oBAC9D,SAAS;gBACX,CAAC;gBACD,MAAM,IAAI,CAAC,QAAQ,CAAC,CAAC;YACvB,CAAC;iBAAM,IAAI,MAAM,EAAE,CAAC;gBAClB,MAAM,GAAG,GAAG,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;gBAEhC,mEAAmE;gBACnE,oEAAoE;gBACpE,IAAI,OAAO,CAAC,GAAG,CAAC,wBAAwB,KAAK,GAAG;oBAC5C,oBAAoB,CAAC,IAAI,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;oBAAE,SAAS;gBAErE,qEAAqE;gBACrE,IAAI,UAAU,IAAI,CAAC,UAAU,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,CAAC;oBAAE,SAAS;gBAEzD,uCAAuC;gBACvC,IAAI,YAAY,IAAI,YAAY,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;oBAC5C,MAAM,OAAO,GAAG,QAAQ,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC;oBAC7C,MAAM,OAAO,GAAG,YAAY,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,CAAC;oBAChE,IAAI,CAAC,OAAO;wBAAE,SAAS;gBACzB,CAAC;gBAED,yBAAyB;gBACzB,IAAI,UAAU,EAAE,CAAC;oBACf,MAAM,OAAO,GAAG,QAAQ,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC;oBAC7C,IAAI,UAAU,CAAC,OAAO,CAAC;wBAAE,SAAS;gBACpC,CAAC;gBAED,gCAAgC;gBAChC,IAAI,CAAC;oBACH,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,CAAC;oBACtC,IAAI,QAAQ,CAAC,IAAI,GAAG,WAAW;wBAAE,SAAS;gBAC5C,CAAC;gBAAC,MAAM,CAAC;oBACP,SAAS;gBACX,CAAC;gBAED,KAAK,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC,CAAC,QAAQ,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC;gBAElE,IAAI,KAAK,CAAC,MAAM,IAAI,QAAQ,EAAE,CAAC;oBAC7B,OAAO,CAAC,IAAI,CACV,qCAAqC,QAAQ,wCAAwC,CACtF,CAAC;oBACF,YAAY,GAAG,IAAI,CAAC;oBACpB,OAAO;gBACT,CAAC;YACH,CAAC;QACH,CAAC;IACH,CAAC;IAED,MAAM,IAAI,CAAC,QAAQ,CAAC,CAAC;IACrB,OAAO,KAAK,CAAC;AACf,CAAC"}
+{"version":3,"file":"walk.js","sourceRoot":"","sources":["../../src/utils/walk.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,WAAW,CAAC;AACrC,OAAO,EAAE,gBAAgB,EAAE,WAAW,EAAE,gBAAgB,EAAE,oBAAoB,EAAE,MAAM,mBAAmB,CAAC;AAC1G,OAAO,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAC;AAC9C,OAAO,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AAC7C,OAAO,EAAE,iBAAiB,EAAE,MAAM,qBAAqB,CAAC;AAGxD,OAAO,EAAE,oBAAoB,EAAE,WAAW,EAAE,gBAAgB,EAAE,CAAC;AAG/D,MAAM,CAAC,KAAK,UAAU,aAAa,CACjC,QAAgB,EAChB,OAAqB;IAErB,MAAM,iBAAiB,GAAG,OAAO,IAAI,EAAE,CAAC;IACxC,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,MAAM,MAAM,GAAG,IAAI,UAAU,CAAC,iBAAiB,CAAC,WAAW,EAAE,iBAAiB,CAAC,QAAQ,CAAC,CAAC;IACzF,MAAM,OAAO,GAAG,gBAAgB,CAAC,QAAQ,EAAE,iBAAiB,CAAC,CAAC;IAE9D,MAAM,iBAAiB,CAAC,QAAQ,EAAE;QAChC,cAAc,EAAE,iBAAiB,CAAC,cAAc,IAAI,KAAK;QACzD,mBAAmB,EAAE,OAAO,CAAC,mBAAmB;QAChD,MAAM,EAAE,KAAK,EAAE,EAAE,QAAQ,EAAE,KAAK,EAAE,KAAK,EAAE,EAAE,EAAE;YAC3C,IAAI,CAAC,OAAO,CAAC,iBAAiB,CAAC,QAAQ,EAAE,KAAK,CAAC,IAAI,CAAC;gBAAE,OAAO,IAAI,CAAC;YAClE,MAAM,SAAS,GAAG,KAAK,IAAI,CAAC,MAAM,SAAS,CAAC,QAAQ,CAAC,CAAC,CAAC;YACvD,IAAI,CAAC,SAAS,IAAI,SAAS,CAAC,IAAI,GAAG,MAAM,CAAC,WAAW;gBAAE,OAAO,IAAI,CAAC;YAEnE,KAAK,CAAC,IAAI,CAAC,iBAAiB,CAAC,QAAQ,CAAC,CAAC,CAAC,QAAQ,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC;YACjF,MAAM,CAAC,UAAU,EAAE,CAAC;YACpB,OAAO,MAAM,CAAC,WAAW,CAAC;QAC5B,CAAC;KACF,CAAC,CAAC;IAEH,OAAO,KAAK,CAAC;AACf,CAAC"}

package/package.json

@@ -1,13 +1,13 @@
 {
   "name": "codesift-mcp",
-  "version": "0.9.5",
+  "version": "0.9.10",
   "description": "MCP server for code intelligence — 146 tools for symbol search, call graph, semantic search, route tracing, community detection, LSP bridge, secret detection, conversation search, and Hono framework intelligence",
   "license": "BSL-1.1",
   "author": "Greg Laskowski",
   "type": "module",
   "bin": {
-    "codesift-mcp": "./dist/server.js",
-    "codesift": "./dist/cli.js"
+    "codesift-mcp": "dist/server.js",
+    "codesift": "dist/cli.js"
   },
   "main": "./dist/server.js",
   "files": [
@@ -19,7 +19,7 @@
     "LICENSE"
   ],
   "scripts": {
-    "build": "tsc && cp -r src/parser/languages dist/parser/",
+    "build": "node -e \"require('node:fs').rmSync('dist',{recursive:true,force:true})\" && tsc && cp -r src/parser/languages dist/parser/ && chmod +x dist/cli.js dist/server.js",
     "dev": "tsx src/server.ts",
     "start": "node dist/server.js",
     "test": "vitest run",
@@ -63,7 +63,8 @@
     "vitest": "^3.0.0"
   },
   "overrides": {
-    "lodash-es": ">=4.18.1"
+    "lodash-es": ">=4.18.1",
+    "esbuild": "0.28.1"
   },
   "engines": {
     "node": ">=20.0.0"