filter-scan-dir 2.1.3 → 2.1.5

package/README.md

@@ -7,7 +7,10 @@
 
 [![npm badge][npm-badge-png]][package-url]
 
-Recursively scan and filter directory for a flat array of files.
+Extremely fast recursive directory crawling and filtering for Node.js.
+Returns a flat array of file paths.
+It can scan 1 million files in about half a second with a warm OS cache and `fullStat: false`.
+See [Performance](#performance) for the measured setup.
 
 - Supports super fast concurrent mode in async version.
 
@@ -34,8 +37,109 @@ console.log(await filterScanDir({ cwd: "test" }));
 
 - **[API Docs]**
 
+# Filtering
+
+| Option | Purpose |
+| --- | --- |
+| `ignoreDirs` | Skip exact directory basenames at every depth. Accepts a string or array. |
+| `prefilter` | Reject entries before `lstat`. Requires `fullStat: true`. |
+| `filterExt` | Include only matching extensions. |
+| `ignoreExt` | Exclude matching extensions. |
+| `filter` | Decide whether to include each file. |
+| `filterDir` | Skip directories before scanning their children. |
+
+`filter` and `filterDir` receive `(name, relativeDir, extras)`.
+Return `true` to accept an entry or `false` to skip it.
+Directories enter the output only with `includeDir: true`.
+Return `{ stop: true }` to stop the scan.
+With `grouping: true`, return a string to choose a result group.
+
+Use `prefilter` when only some entries need full metadata:
+
+```ts
+const files = await filterScanDir({
+  cwd: ".",
+  fullStat: true,
+  ignoreDirs: ["node_modules", ".git"],
+  prefilter: (name, _dir, entry) => entry.isDirectory() || name.endsWith(".ts"),
+  filter: (_name, _dir, { stat }) => stat.size < 100_000,
+});
+```
+
+`prefilter` receives `(name, relativeDir, Dirent)` and runs synchronously.
+Return `false` to reject an entry. Rejecting a directory skips its entire subtree.
+`prefilter` with `fullStat: false` throws before scanning, regardless of `rethrowError`.
+
+The order is `ignoreDirs`, `prefilter`, extension filters, `lstat`, then `filter` or `filterDir`.
+Extension filters apply only to non-directory entries.
+Early rejections avoid `lstat`, so rejected entries cannot report `lstat` errors.
+`ignoreDirs` matches names, not paths or glob patterns. Symlinks are never followed.
+
+# Performance
+
+`fullStat: false` gets entry types from `readdir` as `Dirent` objects.
+This avoids a separate `lstat` call for every file and directory entry.
+In our warm-cache tests, it was **2.3× faster on the [fynmesh repo](https://www.fynmesh.win)** than `fullStat: true`.
+It was **8.1× faster on the 1,000 × 1,000 synthetic tree**.
+
+Use it when names and entry types are enough:
+
+```ts
+const files = await filterScanDir({ cwd: "src", fullStat: false });
+```
+
+Filter callbacks receive `Dirent` objects instead of `Stats` objects.
+Sizes, timestamps, and permissions are unavailable.
+
+The default concurrency is `50`.
+Higher concurrency is not always faster. `concurrency: Infinity` removes the limit.
+More concurrent reads can increase memory use. Benchmark your directory tree before changing it.
+`filterScanDirSync` blocks the event loop.
+
+Keep Dirent mode for filters that only need names or entry types:
+
+```ts
+const files = await filterScanDir({
+  cwd: ".",
+  fullStat: false,
+  filterExt: [".js", ".ts"],
+  ignoreDirs: ["node_modules", ".git"],
+});
+```
+
+- `filterDir` can skip entire subtrees.
+- Extension filters still read every visited directory. Rejected entries skip `lstat` in full-stat mode.
+- Custom callbacks add work per entry.
+- `sortFiles: true` adds sorting work per directory.
+
+Use the default `fullStat: true` for full metadata.
+The extra `lstat` calls can make scans much slower.
+The cost is higher on cold caches or slow storage.
+
+Scan times depend on storage, directory layout, and OS cache state.
+Warm-cache measurements do not predict cold-cache performance.
+Scanning lists entries without reading file contents.
+
+## Example measurements
+
+These are medians from seven warm-cache runs.
+Both modes used concurrency `50`.
+No filters or sorting were enabled. Symlinks were excluded.
+
+Test system: Node.js 22.22.2 on macOS, Apple M4 Pro, 24 GB RAM.
+
+| Tree | Files | Async `fullStat: false` | Async `fullStat: true` |
+| --- | ---: | ---: | ---: |
+| Synthetic: 1,000 directories with 1,000 empty files each | 1,000,000 | 551 ms | 4,483 ms |
+| [fynmesh repo](https://www.fynmesh.win) with installed dependencies: 48,347 directories | 292,242 | 949 ms | 2,157 ms |
+
+The fynmesh repo scan included `node_modules` and `.git`.
+Its directory count includes the root.
+
 # License
 
+Copyright (c) 2022-2026 Joel Chen
+
 Licensed under the [Apache License, Version 2.0](https://www.apache.org/licenses/LICENSE-2.0)
 
 [npm-version-svg]: https://badge.fury.io/js/filter-scan-dir.svg

package/dist/index.d.ts

@@ -2,13 +2,13 @@ import { Dirent, Stats } from "fs";
 /**
  * type of the 3rd argument for the filter callback
  */
-export type ExtrasData = {
+export type ExtrasData<Stat extends Dirent | Stats = Dirent | Stats> = {
     /** name of the file being considered */
     file: string;
     /** path to the directory being processed that contains the file  */
     path: string;
     /** result from fs.lstat or readdir */
-    stat: Dirent | Stats;
+    stat: Stat;
     /** full path with cwd + path + file  */
     fullFile: string;
     /** path to file without cwd: path + file */
@@ -28,7 +28,7 @@ export type FilterInfo = {
     group?: string;
     /** if `true` then skip the file or directory, else add it */
     skip?: boolean;
-    /** stop the scanning and return the result immediately. */
+    /** stop all filtering; async scans drain reads already in flight before returning. */
     stop?: boolean;
     /** if not `undefined`, then use this as the value to add to the output */
     formatName?: string;
@@ -49,11 +49,11 @@ export type FilterResult = boolean | string | FilterInfo;
  *    first level directory.
  * @param extras - extras data
  */
-export type FilterCallback = (file: string, path: string, extras: ExtrasData) => FilterResult;
+export type FilterCallback<Stat extends Dirent | Stats = Dirent | Stats> = (file: string, path: string, extras: ExtrasData<Stat>) => FilterResult;
 /**
  * Options for filterScanDir
  */
-export type Options = {
+export type Options<FullStat extends boolean = boolean> = {
     /** current working directory to start scanning */
     cwd?: string;
     /**
@@ -79,9 +79,10 @@ export type Options = {
      * - *Default*: `true` - for significant performance improvement, set this to `false`
      *
      */
-    fullStat?: boolean;
+    fullStat?: FullStat;
     /**
-     * for async version only - numer of directories to process concurrently. *Default*: `50`
+     * for async version only - maximum directories reading entries or metadata concurrently.
+     * *Default*: `50`. Ancestors waiting for child directories do not consume a slot.
      *
      * - Set this to `0` or `1` to disable concurrent mode
      *
@@ -93,9 +94,16 @@ export type Options = {
     /** set to `true` to throw errors instead of ignoring them */
     rethrowError?: boolean;
     /** callback to filter files. */
-    filter?: FilterCallback;
+    filter?: FilterCallback<FullStat extends false ? Dirent : Stats>;
     /** callback to filter directories. */
-    filterDir?: FilterCallback;
+    filterDir?: FilterCallback<FullStat extends false ? Dirent : Stats>;
+    /** directory basenames to skip at every depth, before reading metadata or children */
+    ignoreDirs?: string | string[];
+    /**
+     * synchronous entry filter before lstat; returning false also prunes directories.
+     * Requires fullStat to be true (the default). Throws when fullStat is false.
+     */
+    prefilter?: (file: string, path: string, entry: Dirent) => boolean;
     /** array or string of extensions to ignore. ext must include `.`, ie: `".js"` */
     ignoreExt?: string | string[];
     /** array or string of extensions to include only, apply after `ignoreExt` */
@@ -107,17 +115,19 @@ export type Options = {
      * - If you didn't specify this, then `cwd` is automatically converted to use `/`.
      */
     pathSep?: string;
-};
+} & ([FullStat] extends [false] ? {
+    fullStat: false;
+} : {});
 /**
  * options specifically to set grouping flag `true` to enable grouping of files
  */
-export type GroupingOptions = {
+export type GroupingOptions<FullStat extends boolean = boolean> = {
     /**
      * enable grouping of files
      * This is default to disabled, so it's only expecting `true` to enable it.
      */
     grouping: true;
-} & Options;
+} & Options<FullStat>;
 /**
  * The scanned result if grouping is enabled.
  *
@@ -133,9 +143,17 @@ export type GroupingResult = {
  * @returns
  */
 export declare function filterScanDir(options?: string): Promise<string[]>;
+export declare function filterScanDir(options: GroupingOptions<true>): Promise<GroupingResult>;
+export declare function filterScanDir(options: GroupingOptions<false>): Promise<GroupingResult>;
+export declare function filterScanDir(options: GroupingOptions): Promise<GroupingResult>;
+export declare function filterScanDir(options: Options<true>): Promise<string[]>;
+export declare function filterScanDir(options: Options<false>): Promise<string[]>;
 export declare function filterScanDir(options?: Options): Promise<string[]>;
-export declare function filterScanDir(options?: GroupingOptions): Promise<GroupingResult>;
 /** sync version of filter scan dir */
 export declare function filterScanDirSync(options?: string): string[];
+export declare function filterScanDirSync(options: GroupingOptions<true>): GroupingResult;
+export declare function filterScanDirSync(options: GroupingOptions<false>): GroupingResult;
+export declare function filterScanDirSync(options: GroupingOptions): GroupingResult;
+export declare function filterScanDirSync(options: Options<true>): string[];
+export declare function filterScanDirSync(options: Options<false>): string[];
 export declare function filterScanDirSync(options?: Options): string[];
-export declare function filterScanDirSync(options?: GroupingOptions): GroupingResult;

package/dist/index.js

@@ -95,12 +95,6 @@ function processFile(options, extras) {
         extras.stat.isSymbolicLink()) {
         return false;
     }
-    if (options.ignoreExt.length > 0 && options.ignoreExt.indexOf(extras.ext) >= 0) {
-        return false;
-    }
-    if (options.filterExt.length > 0 && options.filterExt.indexOf(extras.ext) < 0) {
-        return false;
-    }
     const filterResult = options.filter ? options.filter(extras.file, extras.path, extras) : true; // default to include
     if (filterResult) {
         if (filterResult.skip !== true) {
@@ -110,6 +104,25 @@ function processFile(options, extras) {
     }
     return false;
 }
+// Reject entries before allocating callback extras or requesting full metadata.
+function acceptEntry(options, entry, path) {
+    const isDirectory = entry.isDirectory();
+    if (isDirectory && options._ignoreDirs.has(entry.name))
+        return false;
+    if (options.prefilter && !options.prefilter(entry.name, path, entry))
+        return false;
+    if (isDirectory)
+        return true;
+    if (options.ignoreExt.length || options.filterExt.length) {
+        const ix = entry.name.lastIndexOf(".");
+        const ext = ix > 0 ? entry.name.substring(ix) : "";
+        if (options.ignoreExt.indexOf(ext) >= 0)
+            return false;
+        if (options.filterExt.length && options.filterExt.indexOf(ext) < 0)
+            return false;
+    }
+    return true;
+}
 /**
  * get the result base on grouping enable flag
  *
@@ -118,7 +131,7 @@ function processFile(options, extras) {
  */
 function getResult(options) {
     return options.grouping
-        ? Object.assign({ files: [] }, options.result)
+        ? { files: [], ...options.result }
         : options.result.files || [];
 }
 /**
@@ -135,7 +148,7 @@ function walkSync(path, options, level = 0) {
         const dir = Path.join(options.dir, path);
         let files = Fs.readdirSync(dir, options.readdirOpts);
         if (options.sortFiles) {
-            if (options.fullStat) {
+            if (!options.readdirOpts) {
                 files = files.sort();
             }
             else {
@@ -143,15 +156,20 @@ function walkSync(path, options, level = 0) {
             }
         }
         const dirs = [];
-        let stop = false;
+        const extrasFiles = options.fullStat && options._earlyFilter && (options.filter || options.filterDir)
+            ? files.map((entry) => entry.name)
+            : files;
         // process files first
-        for (let ix = 0; !stop && ix < files.length; ix++) {
+        for (let ix = 0; !options._stopped && ix < files.length; ix++) {
             const file = files[ix];
+            if (options._earlyFilter && !acceptEntry(options, file, path))
+                continue;
             let extras;
             if (options.fullStat) {
-                const fullFile = join2(options._sep, dir, file);
+                const name = options._earlyFilter ? file.name : file;
+                const fullFile = join2(options._sep, dir, name);
                 const stat = Fs.lstatSync(fullFile);
-                extras = makeExtrasData(file, fullFile, path, stat, files, options);
+                extras = makeExtrasData(name, fullFile, path, stat, extrasFiles, options);
             }
             else {
                 const fullFile = join2(options._sep, dir, file.name);
@@ -161,15 +179,16 @@ function walkSync(path, options, level = 0) {
                 dirs.push(extras);
             }
             else {
-                stop = processFile(options, extras);
+                options._stopped = !!processFile(options, extras);
             }
         }
         // now process dirs
-        if (!stop && dirs.length > 0) {
-            for (let ix = 0; ix < dirs.length; ix++) {
+        if (!options._stopped && dirs.length > 0) {
+            for (let ix = 0; !options._stopped && ix < dirs.length; ix++) {
                 const extras = dirs[ix];
                 const flags = processDir(options, extras);
                 if (flags.stop) {
+                    options._stopped = true;
                     break;
                 }
                 if (!flags.skip && level < options.maxLevel) {
@@ -183,10 +202,24 @@ function walkSync(path, options, level = 0) {
             throw err;
         }
     }
-    return getResult(options);
+    return level === 0 ? getResult(options) : undefined;
 }
 const asyncReaddir = Util.promisify(Fs.readdir);
 const asyncLStat = Util.promisify(Fs.lstat);
+// Transfer a directory slot to the next waiter without shifting the queue.
+function releaseDirectory(options) {
+    if (options._waitIndex < options._waiting.length) {
+        const resume = options._waiting[options._waitIndex++];
+        if (options._waitIndex === options._waiting.length) {
+            options._waiting = [];
+            options._waitIndex = 0;
+        }
+        resume();
+    }
+    else {
+        options._concurrentCount--;
+    }
+}
 /**
  * async version of dir walk
  *
@@ -196,12 +229,25 @@ const asyncLStat = Util.promisify(Fs.lstat);
  * @returns
  */
 async function walk(path, options, level = 0) {
+    let promises = [];
+    let hasSlot = false;
     try {
+        if (options.concurrency > 1) {
+            if (options._concurrentCount >= options.concurrency) {
+                await new Promise((resolve) => options._waiting.push(resolve));
+            }
+            else {
+                options._concurrentCount++;
+            }
+            hasSlot = true;
+        }
+        if (options._stopped)
+            return undefined;
         // Use Path.join to normalize the directory path once at entry
         const dir = Path.join(options.dir, path);
         let files = await asyncReaddir(dir, options.readdirOpts);
         if (options.sortFiles) {
-            if (options.fullStat) {
+            if (!options.readdirOpts) {
                 files = files.sort();
             }
             else {
@@ -209,15 +255,22 @@ async function walk(path, options, level = 0) {
             }
         }
         const dirs = [];
-        let stop = false;
+        const extrasFiles = options.fullStat && options._earlyFilter && (options.filter || options.filterDir)
+            ? files.map((entry) => entry.name)
+            : files;
         // process files first
-        for (let ix = 0; !stop && ix < files.length; ix++) {
+        for (let ix = 0; !options._stopped && ix < files.length; ix++) {
             const file = files[ix];
+            if (options._earlyFilter && !acceptEntry(options, file, path))
+                continue;
             let extras;
             if (options.fullStat) {
-                const fullFile = join2(options._sep, dir, file);
+                const name = options._earlyFilter ? file.name : file;
+                const fullFile = join2(options._sep, dir, name);
                 const stat = await asyncLStat(fullFile);
-                extras = makeExtrasData(file, fullFile, path, stat, files, options);
+                if (options._stopped)
+                    break;
+                extras = makeExtrasData(name, fullFile, path, stat, extrasFiles, options);
             }
             else {
                 const fullFile = join2(options._sep, dir, file.name);
@@ -227,33 +280,31 @@ async function walk(path, options, level = 0) {
                 dirs.push(extras);
             }
             else {
-                stop = processFile(options, extras);
+                options._stopped = !!processFile(options, extras);
             }
         }
+        // The slot covers readdir and all lstats, but not waiting for descendants.
+        if (hasSlot) {
+            releaseDirectory(options);
+            hasSlot = false;
+        }
         // now process dirs
-        if (!stop && dirs.length > 0) {
-            let promises = [];
-            for (let ix = 0; ix < dirs.length; ix++) {
+        if (!options._stopped && dirs.length > 0) {
+            for (let ix = 0; !options._stopped && ix < dirs.length; ix++) {
                 const extras = dirs[ix];
                 const flags = processDir(options, extras);
                 if (flags.stop) {
+                    options._stopped = true;
                     break;
                 }
                 if (!flags.skip && level < options.maxLevel) {
                     const walkP = walk(extras.dirFile, options, level + 1);
                     if (options.concurrency > 1) {
-                        if (options._concurrentCount < options.concurrency) {
-                            options._concurrentCount++;
-                            promises.push(walkP);
-                        }
-                        else if (promises.length) {
+                        promises.push(walkP);
+                        // Bound eager child walks as well as active directory operations.
+                        if (promises.length >= options.concurrency) {
                             await Promise.all(promises);
-                            options._concurrentCount -= promises.length;
-                            promises = [walkP];
-                            options._concurrentCount++;
-                        }
-                        else {
-                            await walkP;
+                            promises = [];
                         }
                     }
                     else {
@@ -261,19 +312,27 @@ async function walk(path, options, level = 0) {
                     }
                 }
             }
-            if (promises.length) {
-                await Promise.all(promises);
-                options._concurrentCount -= promises.length;
-                promises = [];
-            }
         }
     }
     catch (err) {
         if (options.rethrowError) {
-            throw err;
+            // Child walks resolve after recording failure, so none can reject unobserved.
+            if (!options._error)
+                options._error = { cause: err };
+            options._stopped = true;
         }
     }
-    return getResult(options);
+    finally {
+        if (hasSlot)
+            releaseDirectory(options);
+        // A callback can throw after siblings have started. Drain them before returning.
+        if (promises.length) {
+            await Promise.all(promises);
+        }
+    }
+    if (level === 0 && options._error)
+        throw options._error.cause;
+    return level === 0 ? getResult(options) : undefined;
 }
 /**
  * make a copy of the user's options with proper defaults
@@ -282,7 +341,11 @@ async function walk(path, options, level = 0) {
  * @returns
  */
 function makeOptions(opts) {
-    const options = typeof opts === "string" ? { cwd: opts } : opts;
+    // Walkers select the matching callback metadata from fullStat at runtime.
+    const options = (typeof opts === "string" ? { cwd: opts } : opts);
+    if (options.prefilter && options.fullStat === false) {
+        throw new TypeError("prefilter requires fullStat: true");
+    }
     const sep = options.pathSep || Path.posix.sep;
     let cwd = options.cwd || options.dir || process.cwd();
     if (!options.hasOwnProperty("pathSep") && cwd.includes("\\")) {
@@ -314,7 +377,8 @@ function makeOptions(opts) {
         grouping: undefined,
     }, options, {
         dir: cwd,
-        result: {},
+        fullStat: options.fullStat === undefined ? true : options.fullStat,
+        result: Object.create(null),
         ignoreExt: []
             .concat(options.ignoreExt)
             .map(cleanExt)
@@ -324,8 +388,19 @@ function makeOptions(opts) {
             .map(cleanExt)
             .filter((x) => x),
         _concurrentCount: 0,
+        _waiting: [],
+        _waitIndex: 0,
+        _stopped: false,
+        _error: undefined,
+        _earlyFilter: false,
+        _ignoreDirs: new Set([].concat(options.ignoreDirs || [])),
     });
-    if (!opts2.fullStat) {
+    opts2._earlyFilter =
+        !!opts2.prefilter ||
+            opts2._ignoreDirs.size > 0 ||
+            opts2.ignoreExt.length > 0 ||
+            opts2.filterExt.length > 0;
+    if (!opts2.fullStat || opts2._earlyFilter) {
         opts2.readdirOpts = { withFileTypes: true };
     }
     return opts2;

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "filter-scan-dir",
-  "version": "2.1.3",
-  "description": "Recursively scan and filter directory for a flat array of files",
+  "version": "2.1.5",
+  "description": "Extremely fast recursive directory crawling and filtering for Node.js.",
   "type": "module",
   "types": "./dist/index.d.ts",
   "main": "./index.cjs",
@@ -62,13 +62,13 @@
   "homepage": "https://github.com/jchip/fynjs/tree/main/packages/filter-scan-dir",
   "dependencies": {},
   "devDependencies": {
-    "@fynjs/run": "^1.1.3",
+    "@fynjs/run": "^1.1.5",
     "@types/node": "^26.4.1",
     "@vitest/coverage-v8": "^5.0.0",
     "@vitest/ui": "^5.0.0",
     "prettier": "^3.5.3",
-    "publish-util": "^3.1.2",
-    "run-verify": "^2.1.3",
+    "publish-util": "^3.1.4",
+    "run-verify": "^2.1.5",
     "typedoc": "^0.28.14",
     "typescript": "^7.0.2",
     "vite": "^8.2.2",