asljs-locator 0.1.2 → 0.1.3

package/README.md

@@ -59,6 +59,11 @@ const files = await resolver.resolve(process.cwd(), {
 `resolve` returns absolute paths, deduplicated and sorted. Give it one location
 or several; several are merged.
 
+The excludes and the filters apply while the tree is walked, not to what it
+found: a folder that `.gitignore` ignores, or whose every file an exclude such
+as `**/node_modules/**` takes away, is not entered at all. That is what keeps
+locating cheap in a tree full of `node_modules` and build output.
+
 `check` answers the same question for a single path without walking the tree,
 which is what makes it usable per file:
 

package/dist/git-ignore.d.ts

@@ -2,16 +2,29 @@ import { Logger } from 'asljs-logging';
 /**
  * Whether a path is ignored by a `.gitignore`.
  *
- * Matchers are loaded from the directory of the path being tested and every
- * directory above it, because a nested `.gitignore` applies to its own subtree
- * and git reads them all. Each directory is read once.
+ * A path is tested against the `.gitignore` of every folder above it, because
+ * a nested `.gitignore` applies to its own subtree and git reads them all.
+ * Each folder's file is read once, and a path is tested only against the
+ * folders it is in, however many other folders were read.
+ *
+ * A path ending with the separator is a folder, tested as `name/`, as git
+ * matches a pattern such as `build/`; `LocationResolver` asks so before it
+ * walks into a folder, and skips the folder when it is ignored, since git
+ * ignores everything inside an ignored folder.
  */
 export declare class GitIgnore {
     private logger;
+    /**
+     * The matcher of each folder read, `null` for a folder without a
+     * `.gitignore`.
+     */
     private matchers;
-    private visitedPaths;
     constructor(logger: Logger);
     isIgnored(targetPath: string): boolean;
     filter(paths: string[]): string[];
-    loadMatchers(directoryPath: string): void;
+    /**
+     * The matcher of the `.gitignore` of a folder, read the first time it is
+     * asked for; `null` when the folder has none.
+     */
+    private matcherOf;
 }

package/dist/git-ignore.js

@@ -1,81 +1,87 @@
 import ignore from 'ignore';
-import { readFileSync, statSync } from 'node:fs';
+import { readFileSync } from 'node:fs';
 import path from 'node:path';
 import { toPosixPath } from './paths.js';
 /**
  * Whether a path is ignored by a `.gitignore`.
  *
- * Matchers are loaded from the directory of the path being tested and every
- * directory above it, because a nested `.gitignore` applies to its own subtree
- * and git reads them all. Each directory is read once.
+ * A path is tested against the `.gitignore` of every folder above it, because
+ * a nested `.gitignore` applies to its own subtree and git reads them all.
+ * Each folder's file is read once, and a path is tested only against the
+ * folders it is in, however many other folders were read.
+ *
+ * A path ending with the separator is a folder, tested as `name/`, as git
+ * matches a pattern such as `build/`; `LocationResolver` asks so before it
+ * walks into a folder, and skips the folder when it is ignored, since git
+ * ignores everything inside an ignored folder.
  */
 export class GitIgnore {
     logger;
+    /**
+     * The matcher of each folder read, `null` for a folder without a
+     * `.gitignore`.
+     */
     matchers;
-    visitedPaths;
     constructor(logger) {
         this.logger = logger;
-        this.matchers = [];
-        this.visitedPaths = new Set();
+        this.matchers = new Map();
     }
     isIgnored(targetPath) {
         if (!path.isAbsolute(targetPath)) {
             throw new Error(`Path must be absolute: ${targetPath}`);
         }
         const isDirectory = targetPath.endsWith(path.sep);
-        const normalisedPath = path.normalize(targetPath);
-        if (isDirectory) {
-            this.loadMatchers(normalisedPath);
-        }
-        else {
-            this.loadMatchers(path.dirname(normalisedPath));
-        }
-        for (const matcher of this.matchers) {
-            const relativePath = toPosixPath(path.relative(matcher.path, normalisedPath));
-            if (relativePath === ''
-                || relativePath.startsWith('../')) {
-                continue;
-            }
-            const testRelativePath = isDirectory
-                ? relativePath + '/'
-                : relativePath;
-            if (matcher.matcher.ignores(testRelativePath)) {
-                return true;
+        const normalisedPath = path.normalize(isDirectory
+            ? targetPath.slice(0, -path.sep.length)
+            : targetPath);
+        let folder = path.dirname(normalisedPath);
+        while (folder !== null) {
+            const matcher = this.matcherOf(folder);
+            if (matcher !== null) {
+                const relativePath = toPosixPath(path.relative(folder, normalisedPath));
+                if (matcher.ignores(isDirectory
+                    ? relativePath + '/'
+                    : relativePath)) {
+                    return true;
+                }
             }
+            const parent = path.dirname(folder);
+            folder =
+                parent === folder
+                    ? null
+                    : parent;
         }
         return false;
     }
     filter(paths) {
         return paths.filter(filePath => !this.isIgnored(filePath));
     }
-    loadMatchers(directoryPath) {
-        if (this.visitedPaths.has(directoryPath)) {
-            return;
+    /**
+     * The matcher of the `.gitignore` of a folder, read the first time it is
+     * asked for; `null` when the folder has none.
+     */
+    matcherOf(folder) {
+        const known = this.matchers.get(folder);
+        if (known !== undefined) {
+            return known;
         }
-        this.visitedPaths
-            .add(directoryPath);
-        const gitIgnorePath = path.join(directoryPath, '.gitignore');
-        let gitIgnoreExists = false;
+        let content = null;
         try {
-            statSync(gitIgnorePath);
-            gitIgnoreExists = true;
+            content =
+                readFileSync(path.join(folder, '.gitignore'), 'utf8');
         }
         catch (err) {
             const code = err.code;
-            if (code !== 'ENOENT') {
+            // A folder without one, or a `.gitignore` that is a folder.
+            if (code !== 'ENOENT'
+                && code !== 'EISDIR') {
                 throw err;
             }
         }
-        if (gitIgnoreExists) {
-            const gitIgnoreContent = readFileSync(gitIgnorePath, 'utf8');
-            const matcher = ignore()
-                .add(gitIgnoreContent);
-            this.matchers.push({ path: directoryPath,
-                matcher });
-        }
-        const parentPath = path.dirname(directoryPath);
-        if (parentPath !== directoryPath) {
-            this.loadMatchers(parentPath);
-        }
+        const matcher = content === null
+            ? null
+            : ignore().add(content);
+        this.matchers.set(folder, matcher);
+        return matcher;
     }
 }

package/dist/location.d.ts

@@ -44,6 +44,17 @@ export declare class LocationResolver {
     check(targetPath: string, basePath: string, location: Location | readonly Location[]): Promise<boolean>;
     private resolveOne;
     private checkOne;
+    /**
+     * Whether `filters` asks for `GitIgnore`, the only filter there is; an
+     * unknown name throws.
+     */
+    private usesGitIgnore;
+    /**
+     * Whether an absolute path matches a pattern, resolved as `kindOf` says:
+     * where it is, from the root, or from the base path. `resolve` and `check`
+     * both match with it, so they agree.
+     */
+    private patternMatcher;
     /**
      * How a pattern resolves: `absolute`, a path of the file system, such as
      * `/tmp/notes/*.md` or `C:/notes/*.md`; `root`, from the root, such as
@@ -67,5 +78,4 @@ export declare class LocationResolver {
      * belonging to a location that never finds it.
      */
     private expand;
-    private applyFilters;
 }

package/dist/location.js

@@ -1,5 +1,5 @@
 import { glob } from 'glob';
-import { minimatch } from 'minimatch';
+import { Minimatch } from 'minimatch';
 import { existsSync } from 'node:fs';
 import path from 'node:path';
 import { GitIgnore } from './git-ignore.js';
@@ -49,7 +49,11 @@ export class LocationResolver {
         const results = [];
         for (const one of toLocations(location)) {
             const resolved = await this.resolveOne(basePath, one);
-            results.push(...resolved);
+            // One at a time: spreading tens of thousands of paths into `push`
+            // exceeds the call stack.
+            for (const match of resolved) {
+                results.push(match);
+            }
         }
         const unique = [...new Set(results)];
         unique.sort((a, b) => a.localeCompare(b));
@@ -80,11 +84,31 @@ export class LocationResolver {
         const normalisedBasePath = path.normalize(path.resolve(basePath));
         assertFilePatterns([...patterns,
             ...exclude]);
-        const matches = new Set(await this.expand(patterns, normalisedBasePath));
-        for (const excluded of await this.expand(exclude, normalisedBasePath)) {
-            matches.delete(excluded);
-        }
-        const result = this.applyFilters([...matches], filters);
+        // Excludes and the filters are applied as the tree is walked, not to what
+        // it found: a folder whose every file is excluded, as by
+        // `**/node_modules/**`, or which `.gitignore` ignores, is not entered.
+        const excluded = exclude.map(pattern => this.patternMatcher(pattern, normalisedBasePath));
+        // `<folder>/**` excludes everything in the folders `<folder>` matches.
+        const excludedFolders = exclude
+            .filter(pattern => pattern.endsWith('/**'))
+            .map(pattern => pattern.slice(0, -'/**'.length))
+            .filter(folder => folder !== ''
+            && folder !== '/')
+            .map(folder => this.patternMatcher(folder, normalisedBasePath));
+        const gitIgnore = this.usesGitIgnore(filters);
+        const ignore = { ignored: (walked) => {
+                const fullPath = walked.fullpath();
+                return excluded.some(matches => matches(fullPath))
+                    || gitIgnore
+                        && this.gitIgnore.isIgnored(fullPath);
+            },
+            childrenIgnored: (walked) => {
+                const fullPath = walked.fullpath();
+                return excludedFolders.some(matches => matches(fullPath))
+                    || gitIgnore
+                        && this.gitIgnore.isIgnored(fullPath + path.sep);
+            } };
+        const result = [...new Set(await this.expand(patterns, normalisedBasePath, ignore))];
         result.sort((a, b) => a.localeCompare(b));
         this.logger.trace('LocationResolver.resolve() => %d matches', result.length);
         return result;
@@ -99,26 +123,42 @@ export class LocationResolver {
             ...exclude]);
         const normalisedTargetPath = path.normalize(path.resolve(targetPath));
         const normalisedBasePath = path.normalize(path.resolve(basePath));
-        const matchesPattern = (pattern) => {
-            const kind = this.kindOf(pattern);
-            if (kind === 'absolute') {
-                return minimatch(toPosixPath(normalisedTargetPath), toPosixPath(pattern), { dot: true });
+        const matches = (pattern) => this.patternMatcher(pattern, normalisedBasePath)(normalisedTargetPath);
+        return patterns.some(matches)
+            && !exclude.some(matches)
+            && !(this.usesGitIgnore(filters)
+                && this.gitIgnore.isIgnored(normalisedTargetPath));
+    }
+    /**
+     * Whether `filters` asks for `GitIgnore`, the only filter there is; an
+     * unknown name throws.
+     */
+    usesGitIgnore(filters) {
+        for (const filter of filters) {
+            if (filter.name !== 'GitIgnore') {
+                throw new Error(`Unknown filter: ${filter.name}`);
             }
-            const relativePath = path.relative(kind === 'root'
-                ? this.rootPath
-                : normalisedBasePath, normalisedTargetPath);
-            return minimatch(relativePath, kind === 'root'
-                ? pattern.slice(1)
-                : pattern, { dot: true });
-        };
-        if (!patterns.some(matchesPattern)) {
-            return false;
         }
-        if (exclude.some(matchesPattern)) {
-            return false;
+        return filters.length > 0;
+    }
+    /**
+     * Whether an absolute path matches a pattern, resolved as `kindOf` says:
+     * where it is, from the root, or from the base path. `resolve` and `check`
+     * both match with it, so they agree.
+     */
+    patternMatcher(pattern, basePath) {
+        const kind = this.kindOf(pattern);
+        if (kind === 'absolute') {
+            const absolute = new Minimatch(toPosixPath(pattern), { dot: true });
+            return targetPath => absolute.match(toPosixPath(targetPath));
         }
-        const result = this.applyFilters([normalisedTargetPath], filters);
-        return result.length > 0;
+        const from = kind === 'root'
+            ? this.rootPath
+            : basePath;
+        const relative = new Minimatch(kind === 'root'
+            ? pattern.slice(1)
+            : pattern, { dot: true });
+        return targetPath => relative.match(toPosixPath(path.relative(from, targetPath)));
     }
     /**
      * How a pattern resolves: `absolute`, a path of the file system, such as
@@ -157,49 +197,44 @@ export class LocationResolver {
      * agree: without it a path such as `.remarkrc.mjs` would be reported as
      * belonging to a location that never finds it.
      */
-    async expand(patterns, basePath) {
+    async expand(patterns, basePath, ignore) {
         const matches = [];
         const absolutePatterns = patterns
             .filter(pattern => this.kindOf(pattern)
             === 'absolute')
             .map(pattern => toPosixPath(pattern));
         if (absolutePatterns.length > 0) {
-            matches.push(...await glob(absolutePatterns, { absolute: true,
+            for (const match of await glob(absolutePatterns, { absolute: true,
                 dot: true,
-                nodir: true }));
+                nodir: true,
+                ignore })) {
+                matches.push(match);
+            }
         }
         const rootPatterns = patterns
             .filter(pattern => this.kindOf(pattern)
             === 'root')
             .map(pattern => pattern.slice(1));
         if (rootPatterns.length > 0) {
-            matches.push(...await glob(rootPatterns, { cwd: this.rootPath,
+            for (const match of await glob(rootPatterns, { cwd: this.rootPath,
                 absolute: true,
                 dot: true,
-                nodir: true }));
+                nodir: true,
+                ignore })) {
+                matches.push(match);
+            }
         }
         const basePatterns = patterns.filter(pattern => this.kindOf(pattern)
             === 'base');
         if (basePatterns.length > 0) {
-            matches.push(...await glob(basePatterns, { cwd: basePath,
+            for (const match of await glob(basePatterns, { cwd: basePath,
                 absolute: true,
                 dot: true,
-                nodir: true }));
-        }
-        return matches;
-    }
-    applyFilters(paths, filters) {
-        let result = paths;
-        for (const filter of filters) {
-            switch (filter.name) {
-                case 'GitIgnore':
-                    result =
-                        this.gitIgnore.filter(result);
-                    break;
-                default:
-                    throw new Error(`Unknown filter: ${filter.name}`);
+                nodir: true,
+                ignore })) {
+                matches.push(match);
             }
         }
-        return result;
+        return matches;
     }
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "asljs-locator",
-  "version": "0.1.2",
+  "version": "0.1.3",
   "description": "Locates files by glob patterns, exclusions and filters such as .gitignore.",
   "type": "module",
   "asljs": {
@@ -18,23 +18,31 @@
     "dist/**",
     "README.md",
     "LICENSE.md",
-    "docs/**"
+    "docs/**",
+    "!dist/*.tsbuildinfo"
   ],
   "scripts": {
     "flint": "toolkit flint",
-    "clean": "toolkit clean",
-    "build": "npm run codegen --if-present && tsc -p tsconfig.build.json",
-    "build:dist": "npm run codegen --if-present && tsc -p tsconfig.dist.json",
+    "clean:build": "node ../../scripts/cleanBuildOutDir.js",
+    "clean:dist": "node ../../scripts/cleanDistOutDir.js",
+    "compile:build": "npm run codegen --if-present && tsc -p tsconfig.build.json",
+    "compile:dist": "npm run codegen --if-present && tsc -p tsconfig.dist.json",
+    "build:dist": "node ../../scripts/buildDistDeps.js && npm run compile:dist",
     "typecheck": "npm run codegen --if-present && tsc -p tsconfig.build.json --noEmit",
     "guard:clean-git": "toolkit ensure-clean-working-directory",
-    "prepack": "npm run clean && npm run build:dist && npm run build && npm run test && npm run guard:clean-git",
+    "prepack": "npm run clean:build && npm run clean:dist && npm run build:dist && npm run compile:build && npm run test && npm run guard:clean-git",
     "postpublish": "toolkit tag-release-revision",
     "test": "node --test --enable-source-maps \"build/**/*.test.js\"",
     "test:watch": "node --watch --test \"build/**/*.test.js\"",
-    "coverage": "npm run build && node --test --experimental-test-coverage --enable-source-maps --test-coverage-exclude=\"**/*.test.js\" \"build/**/*.test.js\"",
+    "coverage": "npm run compile:build && node --test --experimental-test-coverage --enable-source-maps --test-coverage-exclude=\"**/*.test.js\" \"build/**/*.test.js\"",
     "release:patch": "toolkit release-patch",
-    "all": "npm run clean && npm run build:dist && npm run flint && npm run typecheck && npm run build && npm run test"
+    "build": "npm run clean:build && npm run flint && npm run typecheck && npm run compile:build && npm run test"
   },
+  "distDependencies": [
+    "asljs-logging",
+    "asljs-testing",
+    "asljs-tmpdir"
+  ],
   "keywords": [
     "glob",
     "gitignore",