filehound 2.0.0-beta.2 → 2.0.0

package/dist/matchers.js

@@ -0,0 +1,419 @@
+import { parseAge, parseSize } from 'unit-compare';
+import { from } from './arrays.js';
+// Brands predicates that test a file without I/O, so find() can call testSync
+// and save a promise per file. The symbol is not exported from the package, so
+// any other predicate is assumed to need I/O and is tested with test().
+const sync = Symbol('SyncFilePredicate');
+export function isSyncPredicate(predicate) {
+    return predicate[sync] === true;
+}
+// Predicates that need no I/O only implement testSync
+export class AbstractFilePredicate {
+    or(f) {
+        return new Or(this, f);
+    }
+    and(f) {
+        return new And(this, f);
+    }
+    not() {
+        return new Not(this);
+    }
+    async test(file) {
+        return this.testSync(file);
+    }
+}
+export class And extends AbstractFilePredicate {
+    #left;
+    #right;
+    constructor(left, right) {
+        super();
+        this.#left = left;
+        this.#right = right;
+    }
+    get [sync]() {
+        return isSyncPredicate(this.#left) && isSyncPredicate(this.#right);
+    }
+    async test(file) {
+        return await this.#left.test(file) && this.#right.test(file);
+    }
+    testSync(file) {
+        return this.#left.testSync(file) && this.#right.testSync(file);
+    }
+}
+export class Not extends AbstractFilePredicate {
+    #filePredicate;
+    constructor(filePredicate) {
+        super();
+        this.#filePredicate = filePredicate;
+    }
+    get [sync]() {
+        return isSyncPredicate(this.#filePredicate);
+    }
+    async test(file) {
+        return !await this.#filePredicate.test(file);
+    }
+    testSync(file) {
+        return !this.#filePredicate.testSync(file);
+    }
+}
+export class Or extends AbstractFilePredicate {
+    #left;
+    #right;
+    constructor(left, right) {
+        super();
+        this.#left = left;
+        this.#right = right;
+    }
+    get [sync]() {
+        return isSyncPredicate(this.#left) && isSyncPredicate(this.#right);
+    }
+    async test(file) {
+        return await this.#left.test(file) || this.#right.test(file);
+    }
+    testSync(file) {
+        return this.#left.testSync(file) || this.#right.testSync(file);
+    }
+}
+export class Extension extends AbstractFilePredicate {
+    [sync] = true;
+    #extensions;
+    constructor(extensions) {
+        super();
+        this.#extensions = extensions;
+    }
+    testSync(file) {
+        return this.#extensions.includes(file.getPathExtension());
+    }
+}
+export class Glob extends AbstractFilePredicate {
+    [sync] = true;
+    #globPattern;
+    constructor(globPattern) {
+        super();
+        this.#globPattern = globPattern;
+    }
+    testSync(file) {
+        return file.isMatch(this.#globPattern);
+    }
+}
+export class TextMatch extends AbstractFilePredicate {
+    [sync] = true;
+    #pattern;
+    #flags;
+    constructor(pattern, flags) {
+        super();
+        this.#pattern = pattern;
+        this.#flags = flags;
+    }
+    testSync(file) {
+        return new RegExp(this.#pattern, this.#flags).test(file.getName());
+    }
+}
+export class CustomFilter extends AbstractFilePredicate {
+    #fn;
+    constructor(fn) {
+        super();
+        this.#fn = fn;
+    }
+    async test(file) {
+        return this.#fn(file);
+    }
+    testSync(_file) {
+        throw new TypeError('applyFilter() filters can only be used with find(). Use applyFilterSync() with findSync()');
+    }
+}
+export class CustomFilterSync extends AbstractFilePredicate {
+    [sync] = true;
+    #fn;
+    constructor(fn) {
+        super();
+        this.#fn = fn;
+    }
+    testSync(file) {
+        return this.#fn(file);
+    }
+}
+export class Size extends AbstractFilePredicate {
+    #isMatch;
+    constructor(sizeExpression) {
+        super();
+        this.#isMatch = parseSize(sizeExpression);
+    }
+    async test(file) {
+        return this.#isMatch(await file.size());
+    }
+    testSync(file) {
+        return this.#isMatch(file.sizeSync());
+    }
+}
+// Ages are measured when a file is tested, not when the predicate is made, so
+// a predicate kept and reused later still compares against the current time
+const MEASURED_WHEN_TESTED = Object.freeze({ now: () => Date.now() });
+export class Modified extends AbstractFilePredicate {
+    #isMatch;
+    constructor(expression) {
+        super();
+        this.#isMatch = parseAge(expression, MEASURED_WHEN_TESTED);
+    }
+    async test(file) {
+        return this.#isMatch(await file.lastModified());
+    }
+    testSync(file) {
+        return this.#isMatch(file.lastModifiedSync());
+    }
+}
+export class Accessed extends AbstractFilePredicate {
+    #isMatch;
+    constructor(expression) {
+        super();
+        this.#isMatch = parseAge(expression, MEASURED_WHEN_TESTED);
+    }
+    async test(file) {
+        return this.#isMatch(await file.lastAccessed());
+    }
+    testSync(file) {
+        return this.#isMatch(file.lastAccessedSync());
+    }
+}
+export class Changed extends AbstractFilePredicate {
+    #isMatch;
+    constructor(expression) {
+        super();
+        this.#isMatch = parseAge(expression, MEASURED_WHEN_TESTED);
+    }
+    async test(file) {
+        return this.#isMatch(await file.lastChanged());
+    }
+    testSync(file) {
+        return this.#isMatch(file.lastChangedSync());
+    }
+}
+export class MatchAll extends AbstractFilePredicate {
+    [sync] = true;
+    testSync(_file) {
+        return true;
+    }
+}
+export function ext(...args) {
+    const extensions = from(args).map(cleanExtension);
+    return new Extension(extensions);
+}
+/**
+ * Matches every file
+ *
+ * @returns the predicate
+ */
+export function matchAll() {
+    return new MatchAll();
+}
+// The matchers are typed, but JavaScript callers could still pass a number
+// where V1 accepted one, so they're told what to use instead
+function expressionFor(matcher, expression, example) {
+    if (typeof expression !== 'string') {
+        throw new TypeError(`${matcher}() takes an expression, such as '${example}'`);
+    }
+    return expression;
+}
+/**
+ * Matches files by size, using an expression of an optional operator, a
+ * whole number and an optional unit, such as `'>10kb'` or `'<= 1 mb'`.
+ *
+ * The operators are `<`, `<=`, `>`, `>=` and `==` (or `=`), which is the
+ * default. The units, in multiples of 1024, are bytes (`b`, the default),
+ * kilobytes (`k`, `kb`), megabytes (`m`, `mb`), gigabytes (`g`, `gb`) and
+ * terabytes (`t`, `tb`), in any case. Use {@link sizeEquals} for an exact
+ * number of bytes.
+ *
+ * @param expression - the size expression
+ * @returns the predicate
+ * @throws `TypeError`
+ * Thrown if the expression is not a string.
+ * @throws `ExpressionError`
+ * Thrown if the expression can't be parsed or has an unknown unit.
+ * @example
+ * ```ts
+ * FileHound.create().match(size('>1mb'));
+ * ```
+ */
+export function size(expression) {
+    return new Size(expressionFor('size', expression, '>1kb'));
+}
+/**
+ * Matches files of an exact size
+ *
+ * @param bytes - the size in bytes
+ * @returns the predicate
+ * @throws `RangeError`
+ * Thrown if bytes is negative or not an integer.
+ * @example
+ * ```ts
+ * FileHound.create().match(sizeEquals(1024));
+ * ```
+ */
+export function sizeEquals(bytes) {
+    if (!Number.isSafeInteger(bytes) || bytes < 0) {
+        throw new RangeError('Size must be a non-negative integer number of bytes');
+    }
+    return new Size(bytes);
+}
+/**
+ * Matches files with a size of zero bytes
+ *
+ * @returns the predicate
+ */
+export function isEmpty() {
+    return new Size(0);
+}
+/**
+ * Matches files whose path, relative to the search path, satisfies a glob
+ * pattern. Patterns follow Node's `path.matchesGlob` and `fs.glob` rules:
+ * `*.json` matches only the top level of the search path, and a leading `**`
+ * segment matches at any depth. Wildcards, including `**`, do not match names
+ * beginning with a dot.
+ *
+ * @param globPattern - the glob pattern
+ * @returns the predicate
+ * @example
+ * ```ts
+ * FileHound.create().match(glob('*.json'));
+ * ```
+ */
+export function glob(globPattern) {
+    return new Glob(globPattern);
+}
+/**
+ * Matches files using a custom function, which may be asynchronous. Can only
+ * be used with {@link FileHound.find}; use {@link applyFilterSync} with
+ * {@link FileHound.findSync}.
+ *
+ * @param fn - returns, or fulfils with, true for files to include
+ * @returns the predicate
+ * @example
+ * ```ts
+ * FileHound.create().match(applyFilter(async file => await file.size() > 0));
+ * ```
+ */
+export function applyFilter(fn) {
+    return new CustomFilter(fn);
+}
+/**
+ * Matches files using a synchronous custom function. Can be used with both
+ * {@link FileHound.find} and {@link FileHound.findSync}.
+ *
+ * @param fn - returns true for files to include
+ * @returns the predicate
+ * @example
+ * ```ts
+ * FileHound.create().match(applyFilterSync(file => file.sizeSync() > 0));
+ * ```
+ */
+export function applyFilterSync(fn) {
+    return new CustomFilterSync(fn);
+}
+/**
+ * Matches files whose path satisfies a regular expression pattern
+ *
+ * @param pattern - the regular expression pattern
+ * @returns the predicate
+ */
+export function like(pattern) {
+    return new TextMatch(pattern);
+}
+/**
+ * Excludes files whose path satisfies any of the given regular expression patterns
+ *
+ * @remarks
+ * Used with {@link FileHound.match}, a discarded directory is left out of the
+ * results but still searched. To keep the search out of it, pass its name
+ * to {@link FileHound.skipDirectories} instead.
+ *
+ * @param patterns - one or more regular expression patterns
+ * @returns the predicate
+ * @example
+ * ```ts
+ * FileHound.create().match(discard('node_modules', '\\.git'));
+ * ```
+ */
+export function discard(...patterns) {
+    if (patterns.length === 1) {
+        return new TextMatch(patterns[0]).not();
+    }
+    return patterns
+        .map((pattern) => new TextMatch(pattern))
+        .reduce((a, b) => a.or(b))
+        .not();
+}
+function cleanExtension(fileExtension) {
+    if (fileExtension.startsWith('.')) {
+        return fileExtension.slice(1);
+    }
+    return fileExtension;
+}
+/**
+ * Matches files by when they were last modified, using an expression of their
+ * age: an optional operator, a whole number and an optional unit, such as
+ * `'<2 days'`, `'>8h'` or `'>=1 week'`.
+ *
+ * The operators are `<`, `<=`, `>`, `>=` and `==` (or `=`), which is the
+ * default. The units are minutes (`m`, `min`, `mins`, `minute`, `minutes`),
+ * hours (`h`, `hour`, `hours`), days (`d`, `day`, `days`, the default) and
+ * weeks (`w`, `week`, `weeks`), in any case.
+ *
+ * @remarks
+ * Ages are compared in whole units, like `find -mtime`: a file modified two
+ * and a half days ago is 2 days old, so it matches `'=2 days'`, `'>=2 days'`
+ * and `'<3 days'`, but not `'>2 days'`. Days and weeks are calendar days, so
+ * stay correct across daylight saving changes. Ages are measured when each
+ * file is tested.
+ *
+ * @param expression - the age expression
+ * @returns the predicate
+ * @throws `TypeError`
+ * Thrown if the expression is not a string.
+ * @throws `ExpressionError`
+ * Thrown if the expression can't be parsed or has an unknown unit.
+ * @example
+ * ```ts
+ * FileHound.create().match(modified('<2 days'));
+ * ```
+ */
+export function modified(expression) {
+    return new Modified(expressionFor('modified', expression, '<2 days'));
+}
+/**
+ * Matches files by when they were last accessed, using an age expression as
+ * for {@link modified}
+ *
+ * @param expression - the age expression
+ * @returns the predicate
+ * @throws `TypeError`
+ * Thrown if the expression is not a string.
+ * @throws `ExpressionError`
+ * Thrown if the expression can't be parsed or has an unknown unit.
+ */
+export function accessed(expression) {
+    return new Accessed(expressionFor('accessed', expression, '<2 days'));
+}
+/**
+ * Matches files by when their status, such as their permissions, last
+ * changed, using an age expression as for {@link modified}
+ *
+ * @param expression - the age expression
+ * @returns the predicate
+ * @throws `TypeError`
+ * Thrown if the expression is not a string.
+ * @throws `ExpressionError`
+ * Thrown if the expression can't be parsed or has an unknown unit.
+ */
+export function changed(expression) {
+    return new Changed(expressionFor('changed', expression, '<2 days'));
+}
+/**
+ * Negates a predicate
+ *
+ * @param fn - the predicate to negate
+ * @returns a predicate that matches when the given one does not
+ */
+export function not(fn) {
+    return new Not(fn);
+}

package/dist/walker/async.d.ts

@@ -0,0 +1,4 @@
+import { File } from '../file.js';
+import type { FilePredicate } from '../matchers.js';
+import { type WalkOptions } from './walk.js';
+export declare function walkAsync(dir: string, matcher: FilePredicate, opts: WalkOptions): AsyncGenerator<File[]>;

package/dist/walker/async.js

@@ -0,0 +1,237 @@
+import * as fsp from 'node:fs/promises';
+import { File, getChildren, getKnownKind } from '../file.js';
+import { Heap } from './heap.js';
+import { ancestorsOf, assertDirectory, byName, createPathError, EntryType, typeOf, Walk } from './walk.js';
+// How the async walk works, and why
+//
+// The walk yields matches in batches, in the same order as the sync walk:
+// depth first, with siblings in name order. Whoever reads the batches sets the
+// pace, so the walk can't run ahead without limit, and stops when they stop.
+//
+// Walking in order on its own is slow, as each directory would wait on its
+// read. So work is started ahead of the walk, within two limits:
+//
+// - WINDOW: within a directory, the siblings after the one being walked are
+//   visited early, starting their stats, predicates and reads.
+// - READ_AHEAD: across the tree, directories further on are read early.
+//
+// Directories are read ahead in the order the walk will reach them, using
+// each one's Position and a heap. This order matters, and simpler orders were
+// measured and found slow. Node queues reads, so a read the walk needs now
+// waits behind every read started before it. Reading ahead among siblings only
+// lost the concurrency of reading the tree at once (about 30% slower than
+// reading it all, unbounded). Reading ahead in the order reads finished was
+// worse: the walk's reads waited behind reads for far-off directories, and it
+// was only fast once READ_AHEAD exceeded the number of directories in the tree.
+// In walk order, a READ_AHEAD of 32 is enough to match unbounded reading.
+//
+// So before changing the order of reads, measure it with `npm run bench`, on
+// trees both deep and wide.
+//
+// Work started ahead can fail, or be abandoned, before the walk reaches it.
+// prefetch() stops those failures being reported as unhandled, and an aborted
+// walk starts no new reads.
+// How many siblings are visited ahead of the one being walked. Visiting starts
+// the I/O an entry needs: a directory's read, a link's stat or a predicate's.
+// Each level of the walk has its own window, so the work in flight is bounded
+// by this times the depth.
+const WINDOW = 64;
+// How many more directories can be read ahead of the walk, beyond those in
+// its windows, and not yet walked
+const READ_AHEAD = 64;
+// Work started ahead of the walk may fail before the walk reaches it, and
+// later awaits still see the failure. Until then, Node would otherwise report
+// it as unhandled.
+function prefetch(pending) {
+    pending.catch(() => { });
+    return pending;
+}
+// Compared index by index, positions give walk order: depth first, in name order
+function precedes(a, b) {
+    for (let i = 0; i < a.length && i < b.length; i++) {
+        if (a[i] !== b[i]) {
+            return a[i] < b[i];
+        }
+    }
+    return a.length < b.length;
+}
+// Reading directories is most of the work of a walk, so they are read ahead
+// of it, in the order the walk will reach them. Reads are queued, so one read
+// out of order would hold up the walk behind the rest.
+class AsyncWalk extends Walk {
+    #signal;
+    // Read or being read, and not yet taken by the walk
+    #reads = new Map();
+    // Found but not yet read
+    #found = new Heap((a, b) => precedes(a.position, b.position));
+    // Of directories, and links that may be, for when the walk reads them
+    #positions = new WeakMap();
+    #taken = new WeakSet();
+    constructor(matcher, opts) {
+        super(matcher, opts);
+        this.#signal = opts.signal;
+    }
+    // Rejects as soon as the walk is aborted, rather than when the work is done,
+    // which for a slow predicate could be a long wait
+    abortable(pending) {
+        const signal = this.#signal;
+        if (!signal) {
+            return pending;
+        }
+        signal.throwIfAborted();
+        return new Promise((resolve, reject) => {
+            const abort = () => reject(signal.reason);
+            signal.addEventListener('abort', abort, { once: true });
+            pending.then((value) => {
+                signal.removeEventListener('abort', abort);
+                resolve(value);
+            }, (error) => {
+                signal.removeEventListener('abort', abort);
+                reject(error);
+            });
+        });
+    }
+    // For after a yield, as whoever reads the walk may have aborted it meanwhile
+    throwIfAborted() {
+        this.#signal?.throwIfAborted();
+    }
+    // The directory's children in name order, read ahead if it was reached
+    read(dir, depth) {
+        this.#taken.add(dir);
+        const read = this.#reads.get(dir) ?? this.#start({ dir, depth, position: this.#positions.get(dir) ?? [] });
+        this.#reads.delete(dir);
+        this.#readAhead();
+        return read;
+    }
+    #start({ dir, depth, position }) {
+        return prefetch(getChildren(dir).then((children) => {
+            children.sort(byName);
+            this.#find(children, depth + 1, position);
+            return children;
+        }));
+    }
+    // Links are left to the walk, which needs a stat to know if they're directories
+    #find(children, depth, parent) {
+        if (!this.canDescend(depth)) {
+            return;
+        }
+        children.forEach((child, i) => {
+            const kind = getKnownKind(child);
+            if (kind && !kind.isDirectory()) {
+                return;
+            }
+            const position = [...parent, i];
+            this.#positions.set(child, position);
+            if (kind && !this.shouldSkipDirectory(child)) {
+                this.#found.push({ dir: child, depth, position });
+            }
+        });
+        this.#readAhead();
+    }
+    #readAhead() {
+        if (this.#signal?.aborted) {
+            return;
+        }
+        while (this.#reads.size < READ_AHEAD && this.#found.size > 0) {
+            const found = this.#found.pop();
+            if (!this.#taken.has(found.dir)) {
+                this.#reads.set(found.dir, this.#start(found));
+            }
+        }
+    }
+}
+// A linked directory that resolves to one of its own ancestors would be walked forever
+async function isCyclicLink(link) {
+    const target = await fsp.realpath(link.getName());
+    for (const ancestor of ancestorsOf(link)) {
+        if (await fsp.realpath(ancestor) === target) {
+            return true;
+        }
+    }
+    return false;
+}
+// Broken links are skipped, as are all links when ignoreSymlinks is set
+async function isVisitableLink(link, walk) {
+    if (walk.ignoreSymlinks) {
+        return false;
+    }
+    try {
+        return !(await link.isDirectory() && await isCyclicLink(link));
+    }
+    catch {
+        return false;
+    }
+}
+const NOTHING = Object.freeze({});
+// Links have no known kind, so need a stat
+async function visitUntyped(file, walk, depth) {
+    return visitTyped(file, typeOf(await file.getStats()), walk, depth);
+}
+async function visitLink(link, walk, depth) {
+    return await isVisitableLink(link, walk) ? visitUntyped(link, walk, depth) : NOTHING;
+}
+function visitTyped(file, type, walk, depth) {
+    if (type === EntryType.DIRECTORY && walk.shouldSkipDirectory(file)) {
+        return NOTHING;
+    }
+    const children = type === EntryType.DIRECTORY && walk.canDescend(depth) ? walk.read(file, depth) : undefined;
+    if (!walk.isCandidate(file, type)) {
+        return children ? { children } : NOTHING;
+    }
+    if (walk.isSyncMatcher) {
+        return walk.matcher.testSync(file) ? { match: file, children } : children ? { children } : NOTHING;
+    }
+    return walk.matcher.test(file).then(isMatch => ({ match: isMatch ? file : undefined, children }));
+}
+// Children always have a dirent, so only links lack a known kind
+function visitChild(child, walk, depth) {
+    const kind = getKnownKind(child);
+    const visited = kind ? visitTyped(child, typeOf(kind), walk, depth) : visitLink(child, walk, depth);
+    return visited instanceof Promise ? prefetch(visited) : visited;
+}
+// Matches are yielded in batches, in walk order: an entry, then everything
+// below it, then its next sibling, with siblings in name order. A batch is
+// yielded before walking down, so matches arrive as they are found.
+async function* walkChildren(children, walk, depth) {
+    const visits = [];
+    let started = 0;
+    let matches = [];
+    for (let next = 0; next < children.length; next++) {
+        for (const end = Math.min(next + WINDOW, children.length); started < end; started++) {
+            visits[started] = visitChild(children[started], walk, depth);
+        }
+        const visited = visits[next];
+        visits[next] = undefined;
+        const visit = visited instanceof Promise ? await walk.abortable(visited) : visited;
+        if (visit.match) {
+            matches.push(visit.match);
+        }
+        if (visit.children) {
+            if (matches.length > 0) {
+                yield matches;
+                walk.throwIfAborted();
+                matches = [];
+            }
+            yield* walkChildren(await walk.abortable(visit.children), walk, depth + 1);
+        }
+    }
+    if (matches.length > 0) {
+        yield matches;
+        walk.throwIfAborted();
+    }
+}
+export async function* walkAsync(dir, matcher, opts) {
+    opts.signal?.throwIfAborted();
+    const root = new File(dir);
+    let stats;
+    try {
+        stats = await root.getStats();
+    }
+    catch (error) {
+        throw createPathError(dir, error);
+    }
+    assertDirectory(dir, stats);
+    // The root is never a match, only the place to search, so the walk starts at its children
+    const walk = new AsyncWalk(matcher, opts);
+    yield* walkChildren(await walk.abortable(walk.read(root, 0)), walk, 1);
+}

package/dist/walker/heap.d.ts

@@ -0,0 +1,7 @@
+export declare class Heap<T> {
+    #private;
+    constructor(precedes: (a: T, b: T) => boolean);
+    get size(): number;
+    push(item: T): void;
+    pop(): T | undefined;
+}