filehound 2.0.0-beta.2 → 2.0.0-beta.3

package/dist/walker/async.js

@@ -0,0 +1,237 @@
+import * as fsp from 'node:fs/promises';
+import { File, getChildren, getKnownKind } from '../file.js';
+import { Heap } from './heap.js';
+import { ancestorsOf, assertDirectory, byName, createPathError, EntryType, typeOf, Walk } from './walk.js';
+// How the async walk works, and why
+//
+// The walk yields matches in batches, in the same order as the sync walk:
+// depth first, with siblings in name order. Whoever reads the batches sets the
+// pace, so the walk can't run ahead without limit, and stops when they stop.
+//
+// Walking in order on its own is slow, as each directory would wait on its
+// read. So work is started ahead of the walk, within two limits:
+//
+// - WINDOW: within a directory, the siblings after the one being walked are
+//   visited early, starting their stats, predicates and reads.
+// - READ_AHEAD: across the tree, directories further on are read early.
+//
+// Directories are read ahead in the order the walk will reach them, using
+// each one's Position and a heap. This order matters, and simpler orders were
+// measured and found slow. Node queues reads, so a read the walk needs now
+// waits behind every read started before it. Reading ahead among siblings only
+// lost the concurrency of reading the tree at once (about 30% slower than
+// reading it all, unbounded). Reading ahead in the order reads finished was
+// worse: the walk's reads waited behind reads for far-off directories, and it
+// was only fast once READ_AHEAD exceeded the number of directories in the tree.
+// In walk order, a READ_AHEAD of 32 is enough to match unbounded reading.
+//
+// So before changing the order of reads, measure it with `npm run bench`, on
+// trees both deep and wide.
+//
+// Work started ahead can fail, or be abandoned, before the walk reaches it.
+// prefetch() stops those failures being reported as unhandled, and an aborted
+// walk starts no new reads.
+// How many siblings are visited ahead of the one being walked. Visiting starts
+// the I/O an entry needs: a directory's read, a link's stat or a predicate's.
+// Each level of the walk has its own window, so the work in flight is bounded
+// by this times the depth.
+const WINDOW = 64;
+// How many more directories can be read ahead of the walk, beyond those in
+// its windows, and not yet walked
+const READ_AHEAD = 64;
+// Work started ahead of the walk may fail before the walk reaches it, and
+// later awaits still see the failure. Until then, Node would otherwise report
+// it as unhandled.
+function prefetch(pending) {
+    pending.catch(() => { });
+    return pending;
+}
+// Compared index by index, positions give walk order: depth first, in name order
+function precedes(a, b) {
+    for (let i = 0; i < a.length && i < b.length; i++) {
+        if (a[i] !== b[i]) {
+            return a[i] < b[i];
+        }
+    }
+    return a.length < b.length;
+}
+// Reading directories is most of the work of a walk, so they are read ahead
+// of it, in the order the walk will reach them. Reads are queued, so one read
+// out of order would hold up the walk behind the rest.
+class AsyncWalk extends Walk {
+    #signal;
+    // Read or being read, and not yet taken by the walk
+    #reads = new Map();
+    // Found but not yet read
+    #found = new Heap((a, b) => precedes(a.position, b.position));
+    // Of directories, and links that may be, for when the walk reads them
+    #positions = new WeakMap();
+    #taken = new WeakSet();
+    constructor(matcher, opts) {
+        super(matcher, opts);
+        this.#signal = opts.signal;
+    }
+    // Rejects as soon as the walk is aborted, rather than when the work is done,
+    // which for a slow predicate could be a long wait
+    abortable(pending) {
+        const signal = this.#signal;
+        if (!signal) {
+            return pending;
+        }
+        signal.throwIfAborted();
+        return new Promise((resolve, reject) => {
+            const abort = () => reject(signal.reason);
+            signal.addEventListener('abort', abort, { once: true });
+            pending.then((value) => {
+                signal.removeEventListener('abort', abort);
+                resolve(value);
+            }, (error) => {
+                signal.removeEventListener('abort', abort);
+                reject(error);
+            });
+        });
+    }
+    // For after a yield, as whoever reads the walk may have aborted it meanwhile
+    throwIfAborted() {
+        this.#signal?.throwIfAborted();
+    }
+    // The directory's children in name order, read ahead if it was reached
+    read(dir, depth) {
+        this.#taken.add(dir);
+        const read = this.#reads.get(dir) ?? this.#start({ dir, depth, position: this.#positions.get(dir) ?? [] });
+        this.#reads.delete(dir);
+        this.#readAhead();
+        return read;
+    }
+    #start({ dir, depth, position }) {
+        return prefetch(getChildren(dir).then((children) => {
+            children.sort(byName);
+            this.#find(children, depth + 1, position);
+            return children;
+        }));
+    }
+    // Links are left to the walk, which needs a stat to know if they're directories
+    #find(children, depth, parent) {
+        if (!this.canDescend(depth)) {
+            return;
+        }
+        children.forEach((child, i) => {
+            const kind = getKnownKind(child);
+            if (kind && !kind.isDirectory()) {
+                return;
+            }
+            const position = [...parent, i];
+            this.#positions.set(child, position);
+            if (kind && !this.shouldSkipDirectory(child)) {
+                this.#found.push({ dir: child, depth, position });
+            }
+        });
+        this.#readAhead();
+    }
+    #readAhead() {
+        if (this.#signal?.aborted) {
+            return;
+        }
+        while (this.#reads.size < READ_AHEAD && this.#found.size > 0) {
+            const found = this.#found.pop();
+            if (!this.#taken.has(found.dir)) {
+                this.#reads.set(found.dir, this.#start(found));
+            }
+        }
+    }
+}
+// A linked directory that resolves to one of its own ancestors would be walked forever
+async function isCyclicLink(link) {
+    const target = await fsp.realpath(link.getName());
+    for (const ancestor of ancestorsOf(link)) {
+        if (await fsp.realpath(ancestor) === target) {
+            return true;
+        }
+    }
+    return false;
+}
+// Broken links are skipped, as are all links when ignoreSymlinks is set
+async function isVisitableLink(link, walk) {
+    if (walk.ignoreSymlinks) {
+        return false;
+    }
+    try {
+        return !(await link.isDirectory() && await isCyclicLink(link));
+    }
+    catch {
+        return false;
+    }
+}
+const NOTHING = Object.freeze({});
+// Links have no known kind, so need a stat
+async function visitUntyped(file, walk, depth) {
+    return visitTyped(file, typeOf(await file.getStats()), walk, depth);
+}
+async function visitLink(link, walk, depth) {
+    return await isVisitableLink(link, walk) ? visitUntyped(link, walk, depth) : NOTHING;
+}
+function visitTyped(file, type, walk, depth) {
+    if (type === EntryType.DIRECTORY && walk.shouldSkipDirectory(file)) {
+        return NOTHING;
+    }
+    const children = type === EntryType.DIRECTORY && walk.canDescend(depth) ? walk.read(file, depth) : undefined;
+    if (!walk.isCandidate(file, type)) {
+        return children ? { children } : NOTHING;
+    }
+    if (walk.isSyncMatcher) {
+        return walk.matcher.testSync(file) ? { match: file, children } : children ? { children } : NOTHING;
+    }
+    return walk.matcher.test(file).then(isMatch => ({ match: isMatch ? file : undefined, children }));
+}
+// Children always have a dirent, so only links lack a known kind
+function visitChild(child, walk, depth) {
+    const kind = getKnownKind(child);
+    const visited = kind ? visitTyped(child, typeOf(kind), walk, depth) : visitLink(child, walk, depth);
+    return visited instanceof Promise ? prefetch(visited) : visited;
+}
+// Matches are yielded in batches, in walk order: an entry, then everything
+// below it, then its next sibling, with siblings in name order. A batch is
+// yielded before walking down, so matches arrive as they are found.
+async function* walkChildren(children, walk, depth) {
+    const visits = [];
+    let started = 0;
+    let matches = [];
+    for (let next = 0; next < children.length; next++) {
+        for (const end = Math.min(next + WINDOW, children.length); started < end; started++) {
+            visits[started] = visitChild(children[started], walk, depth);
+        }
+        const visited = visits[next];
+        visits[next] = undefined;
+        const visit = visited instanceof Promise ? await walk.abortable(visited) : visited;
+        if (visit.match) {
+            matches.push(visit.match);
+        }
+        if (visit.children) {
+            if (matches.length > 0) {
+                yield matches;
+                walk.throwIfAborted();
+                matches = [];
+            }
+            yield* walkChildren(await walk.abortable(visit.children), walk, depth + 1);
+        }
+    }
+    if (matches.length > 0) {
+        yield matches;
+        walk.throwIfAborted();
+    }
+}
+export async function* walkAsync(dir, matcher, opts) {
+    opts.signal?.throwIfAborted();
+    const root = new File(dir);
+    let stats;
+    try {
+        stats = await root.getStats();
+    }
+    catch (error) {
+        throw createPathError(dir, error);
+    }
+    assertDirectory(dir, stats);
+    // The root is never a match, only the place to search, so the walk starts at its children
+    const walk = new AsyncWalk(matcher, opts);
+    yield* walkChildren(await walk.abortable(walk.read(root, 0)), walk, 1);
+}

package/dist/walker/heap.d.ts

@@ -0,0 +1,7 @@
+export declare class Heap<T> {
+    #private;
+    constructor(precedes: (a: T, b: T) => boolean);
+    get size(): number;
+    push(item: T): void;
+    pop(): T | undefined;
+}

package/dist/walker/heap.js

@@ -0,0 +1,49 @@
+// A binary min-heap: pop returns the item that comes first by precedes
+export class Heap {
+    #items = [];
+    #precedes;
+    constructor(precedes) {
+        this.#precedes = precedes;
+    }
+    get size() {
+        return this.#items.length;
+    }
+    push(item) {
+        const items = this.#items;
+        let i = items.push(item) - 1;
+        while (i > 0) {
+            const parent = (i - 1) >> 1;
+            if (!this.#precedes(items[i], items[parent])) {
+                break;
+            }
+            [items[i], items[parent]] = [items[parent], items[i]];
+            i = parent;
+        }
+    }
+    pop() {
+        const items = this.#items;
+        const first = items[0];
+        const last = items.pop();
+        if (items.length > 0 && last !== undefined) {
+            items[0] = last;
+            let i = 0;
+            for (;;) {
+                const left = 2 * i + 1;
+                const right = left + 1;
+                let next = i;
+                if (left < items.length && this.#precedes(items[left], items[next])) {
+                    next = left;
+                }
+                if (right < items.length && this.#precedes(items[right], items[next])) {
+                    next = right;
+                }
+                if (next === i) {
+                    break;
+                }
+                [items[i], items[next]] = [items[next], items[i]];
+                i = next;
+            }
+        }
+        return first;
+    }
+}

package/dist/walker/sync.d.ts

@@ -0,0 +1,4 @@
+import { File } from '../file.js';
+import type { FilePredicate } from '../matchers.js';
+import { type WalkOptions } from './walk.js';
+export declare function walkSync(dir: string, matcher: FilePredicate, opts: WalkOptions): File[];

package/dist/walker/sync.js

@@ -0,0 +1,66 @@
+import * as fs from 'node:fs';
+import { File, getChildrenSync, getKnownKind } from '../file.js';
+import { ancestorsOf, assertDirectory, byName, createPathError, EntryType, typeOf, Walk } from './walk.js';
+// A linked directory that resolves to one of its own ancestors would be walked forever
+function isCyclicLink(link) {
+    const target = fs.realpathSync(link.getName());
+    return ancestorsOf(link).some(ancestor => fs.realpathSync(ancestor) === target);
+}
+// Broken links are skipped, as are all links when ignoreSymlinks is set
+function isVisitableLink(link, walk) {
+    if (walk.ignoreSymlinks) {
+        return false;
+    }
+    try {
+        return !(link.isDirectorySync() && isCyclicLink(link));
+    }
+    catch {
+        return false;
+    }
+}
+// Links have no known kind, so need a stat
+function visitUntyped(file, walk, depth, matches) {
+    return visitTyped(file, typeOf(file.getStatsSync()), walk, depth, matches);
+}
+function visitLink(link, walk, depth, matches) {
+    return isVisitableLink(link, walk) ? visitUntyped(link, walk, depth, matches) : matches;
+}
+// Entries are visited depth first in name order, adding to a single list of matches
+function visitTyped(file, type, walk, depth, matches) {
+    if (type === EntryType.DIRECTORY && walk.shouldSkipDirectory(file)) {
+        return matches;
+    }
+    if (walk.isCandidate(file, type) && walk.matcher.testSync(file)) {
+        matches.push(file);
+    }
+    if (type === EntryType.DIRECTORY && walk.canDescend(depth)) {
+        visitChildren(file, walk, depth + 1, matches);
+    }
+    return matches;
+}
+// Children always have a dirent, so only links lack a known kind
+function visitChildren(dir, walk, depth, matches) {
+    for (const child of getChildrenSync(dir).sort(byName)) {
+        const kind = getKnownKind(child);
+        if (kind) {
+            visitTyped(child, typeOf(kind), walk, depth, matches);
+        }
+        else {
+            visitLink(child, walk, depth, matches);
+        }
+    }
+    return matches;
+}
+export function walkSync(dir, matcher, opts) {
+    const root = new File(dir);
+    let stats;
+    try {
+        stats = root.getStatsSync();
+    }
+    catch (error) {
+        throw createPathError(dir, error);
+    }
+    assertDirectory(dir, stats);
+    // The root is never a match, only the place to search, so the walk starts at its children
+    return visitChildren(root, new Walk(matcher, opts), 1, []);
+}

package/dist/walker/walk.d.ts

@@ -0,0 +1,32 @@
+import type { File, FileKind } from '../file.js';
+import { FileType } from '../filetype.js';
+import { type FilePredicate } from '../matchers.js';
+export type WalkOptions = {
+    maxDepth?: number;
+    ignoreHiddenFiles: boolean;
+    ignoreHiddenDirectories: boolean;
+    ignoreSymlinks: boolean;
+    fileTypes: FileType[];
+    signal?: AbortSignal;
+};
+export declare const EntryType: {
+    readonly OTHER: "other";
+    readonly REGULAR: "regular";
+    readonly DIRECTORY: "directory";
+};
+export type EntryType = typeof EntryType[keyof typeof EntryType];
+export declare function createPathError(name: string, cause: unknown): Error;
+export declare function assertDirectory(name: string, stats: FileKind): void;
+export declare class Walk {
+    #private;
+    readonly matcher: FilePredicate;
+    readonly isSyncMatcher: boolean;
+    constructor(matcher: FilePredicate, opts: WalkOptions);
+    get ignoreSymlinks(): boolean;
+    isCandidate(file: File, type: EntryType): boolean;
+    shouldSkipDirectory(dir: File): boolean;
+    canDescend(depth: number): boolean;
+}
+export declare function typeOf(kind: FileKind): EntryType;
+export declare function byName(a: File, b: File): number;
+export declare function ancestorsOf(file: File): string[];

package/dist/walker/walk.js

@@ -0,0 +1,70 @@
+import { dirname, resolve } from 'node:path';
+import { FileType } from '../filetype.js';
+import { isSyncPredicate } from '../matchers.js';
+// Every entry has one. FileType is the public subset; anything else, such as a
+// socket or FIFO, is OTHER and never matched.
+export const EntryType = { ...FileType, OTHER: 'other' };
+export function createPathError(name, cause) {
+    return new Error(`'${name}' is not a valid path`, { cause });
+}
+// Search paths are where to search, so must be directories
+export function assertDirectory(name, stats) {
+    if (!stats.isDirectory()) {
+        throw new Error(`'${name}' is not a directory`);
+    }
+}
+// State shared by a walk. The async and sync walks (async.ts and sync.ts) share
+// everything except the calls that touch the file system, and both return
+// matches in name order.
+export class Walk {
+    matcher;
+    // Checked once, as combined predicates work it out from their parts
+    isSyncMatcher;
+    #opts;
+    constructor(matcher, opts) {
+        this.matcher = matcher;
+        this.isSyncMatcher = isSyncPredicate(matcher);
+        this.#opts = opts;
+    }
+    get ignoreSymlinks() {
+        return this.#opts.ignoreSymlinks;
+    }
+    isCandidate(file, type) {
+        switch (type) {
+            case EntryType.DIRECTORY:
+                return this.#typeEnabled(type);
+            case EntryType.REGULAR:
+                return this.#typeEnabled(type) && !(this.#opts.ignoreHiddenFiles && file.isHiddenSync());
+            case EntryType.OTHER:
+                return false;
+        }
+    }
+    // Hidden directories are neither returned nor walked
+    shouldSkipDirectory(dir) {
+        return this.#opts.ignoreHiddenDirectories && dir.isHiddenSync();
+    }
+    // depth counts levels below the root, which is 0. A directory beyond
+    // maxDepth is still returned, like any other entry at its level, but not walked.
+    canDescend(depth) {
+        return this.#opts.maxDepth === undefined || depth <= this.#opts.maxDepth;
+    }
+    #typeEnabled(type) {
+        return this.#opts.fileTypes.includes(type);
+    }
+}
+export function typeOf(kind) {
+    return kind.isDirectory() ? EntryType.DIRECTORY : kind.isFile() ? EntryType.REGULAR : EntryType.OTHER;
+}
+export function byName(a, b) {
+    return a.getName() < b.getName() ? -1 : a.getName() > b.getName() ? 1 : 0;
+}
+export function ancestorsOf(file) {
+    const ancestors = [];
+    let ancestor = dirname(resolve(file.getName()));
+    while (dirname(ancestor) !== ancestor) {
+        ancestors.push(ancestor);
+        ancestor = dirname(ancestor);
+    }
+    ancestors.push(ancestor);
+    return ancestors;
+}

package/package.json

@@ -1,30 +1,44 @@
 {
   "name": "filehound",
-  "version": "2.0.0-beta.2",
+  "version": "2.0.0-beta.3",
   "description": "Find files the easy way",
-  "main": "index.js",
+  "type": "module",
+  "main": "dist/index.js",
+  "types": "dist/index.d.ts",
+  "exports": {
+    ".": {
+      "types": "./dist/index.d.ts",
+      "default": "./dist/index.js"
+    }
+  },
+  "files": [
+    "dist"
+  ],
+  "engines": {
+    "node": ">=24"
+  },
   "scripts": {
-    "jsdoc": "jsdoc -c ./docconfig/jsdoc.json",
+    "docs": "typedoc",
     "build": "tsc",
+    "typecheck": "tsc -p tsconfig.test.json",
     "watch": "tsc --watch",
-    "clean": "rm -fr dist/*",
-    "prepublish": "npm run build",
-    "prepublishOnly": "tsc -p ./ --outDir dist",
-    "test": "mocha -r ts-node/register test/*.ts",
+    "clean": "node -e \"require('fs').rmSync('dist', { recursive: true, force: true })\"",
+    "prepublishOnly": "npm run clean && npm run build",
+    "test": "mocha",
     "posttest": "npm run lint",
-    "lint": "tslint --project tsconfig.json",
-    "coverage": "nyc mocha && nyc report --reporter=html && nyc report --reporter=json-summary"
+    "lint": "eslint src test bench",
+    "coverage": "c8 mocha",
+    "bench": "npm run build && node bench/bench.mjs"
   },
   "repository": {
     "type": "git",
-    "url": "https://github.com/nspragg/filehound.git"
+    "url": "git+https://github.com/nspragg/filehound.git"
   },
   "keywords": [
     "file",
     "recursive",
     "search",
     "filehound",
-    "file",
     "hound",
     "lister",
     "find",
@@ -36,48 +50,30 @@
     "fs",
     "file system"
   ],
+  "author": "nspragg <nspragg@gmail.com>",
   "license": "MIT",
   "bugs": {
     "url": "https://github.com/nspragg/filehound/issues"
   },
   "homepage": "https://github.com/nspragg/filehound",
-  "devDependencies": {
-    "@types/mocha": "^2.2.43",
-    "@types/node": "^8.0.33",
-    "@types/typescript": "^2.0.0",
-    "eslint-config-iplayer-base": "^1.0.0",
-    "chai": "^4.1.2",
-    "coveralls": "^2.11.9",
-    "eslint": "^3.11.1",
-    "istanbul": "1.1.0-alpha.1",
-    "jsdoc": "^3.4.3",
-    "mocha": "^5.0.5",
-    "sinon": "^2.1.0",
-    "ts-node": "^3.3.0",
-    "tslint": "^5.8.0",
-    "tslint-config-airbnb": "^5.3.1",
-    "tslint-eslint-rules": "^4.1.1",
-    "typescript": "^2.6.1"
-  },
-  "nyc": {
-    "include": [
-      "src/**/*.ts"
-    ],
-    "extension": [
-      ".ts"
-    ],
-    "require": [
-      "ts-node/register"
-    ],
-    "sourceMap": true,
-    "instrument": true
-  },
   "dependencies": {
-    "file-js": "0.3.0",
-    "lodash": "^4.17.5",
-    "minimatch": "^3.0.4",
-    "moment": "^2.22.0",
-    "npm": "^5.8.0",
-    "unit-compare": "^1.0.1"
+    "unit-compare": "^2.0.0"
+  },
+  "devDependencies": {
+    "@eslint/js": "^10.0.1",
+    "@stylistic/eslint-plugin": "^5.10.0",
+    "@types/chai": "^5.2.3",
+    "@types/mocha": "^10.0.10",
+    "@types/node": "^24.19.2",
+    "@types/sinon": "^22.0.2",
+    "c8": "^12.0.0",
+    "chai": "^6.3.0",
+    "eslint": "^10.13.0",
+    "mocha": "^12.0.3",
+    "sinon": "^22.1.0",
+    "tsx": "^4.23.15",
+    "typedoc": "^0.28.20",
+    "typescript": "~6.0",
+    "typescript-eslint": "^8.72.1"
   }
 }

package/.github/stale.yml

@@ -1,42 +0,0 @@
-# Configuration for probot-stale - https://github.com/probot/stale
-
-# Number of days of inactivity before an Issue or Pull Request becomes stale
-daysUntilStale: 5
-# Number of days of inactivity before a stale Issue or Pull Request is closed.
-# Set to false to disable. If disabled, issues still need to be closed manually, but will remain marked as stale.
-daysUntilClose: 7
-# Issues or Pull Requests with these labels will never be considered stale. Set to `[]` to disable
-exemptLabels:
-  - pinned
-  - security
-  - "[Status] Maybe Later"
-# Set to true to ignore issues in a project (defaults to false)
-exemptProjects: false
-# Set to true to ignore issues in a milestone (defaults to false)
-exemptMilestones: false
-# Label to use when marking as stale
-staleLabel: wontfix
-# Comment to post when marking as stale. Set to `false` to disable
-markComment: >
-  This issue has been automatically marked as stale because it has not had
-  recent activity. It will be closed if no further activity occurs. Thank you
-  for your contributions.
-# Comment to post when removing the stale label. Set to `false` to disable
-unmarkComment: false
-# Comment to post when closing a stale Issue or Pull Request. Set to `false` to disable
-closeComment: false
-# Limit the number of actions per hour, from 1-30. Default is 30
-limitPerRun: 30
-# Limit to only `issues` or `pulls`
-# only: issues
-#
-# Optionally, specify configuration settings that are specific to just 'issues' or 'pulls':
-# pulls:
-#   daysUntilStale: 30
-#   markComment: >
-#     This pull request has been automatically marked as stale because it has not had
-#     recent activity. It will be closed if no further activity occurs. Thank you
-#     for your contributions.
-# issues:
-#   exemptLabels:
-#     - confirmed

package/.travis.yml

@@ -1,8 +0,0 @@
-language: node_js
-os:
-  - linux
-node_js:
-  - "8"
-after_script:
-  - npm run coverage
-  - cat coverage/lcov.info | node_modules/.bin/coveralls

package/.vscode/launch.json

@@ -1,21 +0,0 @@
-{
-    // Use IntelliSense to learn about possible Node.js debug attributes.
-    // Hover to view descriptions of existing attributes.
-    // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
-    "version": "0.2.0",
-    "configurations": [
-        {
-            "type": "node",
-            "request": "launch",
-            "name": "Launch Program",
-            "program": "${workspaceRoot}/index.js"
-        },
-        {
-            "type": "node",
-            "request": "attach",
-            "name": "Attach to Process",
-            "address": "localhost",
-            "port": 5858
-        }
-    ]
-}

package/.vscode/settings.json

@@ -1,3 +0,0 @@
-// Place your settings in this file to overwrite default and user settings.
-{
-}