@fynpo/base 1.1.22 → 2.0.0

package/dist/index.js

@@ -1,19 +1,17 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.caching = void 0;
-exports.readFynpoPackages = readFynpoPackages;
-exports.makePkgDeps = makePkgDeps;
-const tslib_1 = require("tslib");
-const path_1 = tslib_1.__importDefault(require("path"));
-const fs_1 = require("fs");
-const filter_scan_dir_1 = require("filter-scan-dir");
-const minimatch_1 = tslib_1.__importDefault(require("minimatch"));
-const lodash_1 = tslib_1.__importDefault(require("lodash"));
-const minimatch_group_1 = require("./minimatch-group");
-tslib_1.__exportStar(require("./fynpo-dep-graph"), exports);
-tslib_1.__exportStar(require("./fynpo-config"), exports);
-tslib_1.__exportStar(require("./util"), exports);
-exports.caching = tslib_1.__importStar(require("./caching"));
+import Path from "path";
+import { promises as Fs } from "fs";
+import { filterScanDir } from "filter-scan-dir";
+import { Minimatch } from "minimatch";
+import _ from "lodash";
+import { groupMM } from "./minimatch-group.js";
+import { resolvePackagesConfig, scanPatterns, includeFilter, outOfScopePackages, } from "./packages-config.js";
+import { makeGitignoreMatcher } from "./gitignore.js";
+export * from "./fynpo-dep-graph.js";
+export * from "./fynpo-config.js";
+export * from "./packages-config.js";
+export * from "./gitignore.js";
+export * from "./util.js";
+export * as caching from "./caching.js";
 /**
  * Take an array of packages and figure out their dependencies on each other
  *
@@ -22,7 +20,7 @@ exports.caching = tslib_1.__importStar(require("./caching"));
 function processDirectDeps(packages) {
     const add = (name, deps, type) => {
         const depPkg = packages[name];
-        lodash_1.default.each(deps, (semver, depName) => {
+        _.each(deps, (semver, depName) => {
             if (!packages.hasOwnProperty(depName)) {
                 return;
             }
@@ -31,7 +29,7 @@ function processDirectDeps(packages) {
             depPkg.localDepsByType[type].push(depName);
         });
     };
-    lodash_1.default.each(packages, (pkg, name) => {
+    _.each(packages, (pkg, name) => {
         add(name, pkg.dependencies, "dep");
         add(name, pkg.devDependencies, "dev");
         add(name, pkg.optionalDependencies, "opt");
@@ -44,31 +42,81 @@ function processDirectDeps(packages) {
  * @param circulars - array of package pairs that depend on each other
  */
 function processIndirectDeps(packages, circulars) {
-    let change = 0;
-    const add = (info, deps) => {
-        lodash_1.default.each(deps, (dep) => {
+    //
+    // Membership used to be `Array.indexOf` over localDeps / indirectDeps / circulars, and
+    // both the walk and the fixpoint were recursive. That is what made a cycle fatal before
+    // FPO-19, and it stayed expensive afterwards: a 500-package chain took ~17s to resolve,
+    // and a deep enough graph could still exhaust the stack through the walk alone, cycle or
+    // no cycle. Sets for membership and explicit stacks for both loops (FPO-43).
+    //
+    // The arrays are still the output, appended in the same discovery order as before - the
+    // Sets only answer "already have it?" without an O(n) scan.
+    //
+    const localDepsSet = new Map();
+    const indirectDepsSet = new Map();
+    const circularsSet = new Set(circulars);
+    _.each(packages, (pkg, name) => {
+        localDepsSet.set(name, new Set(pkg.localDeps));
+        indirectDepsSet.set(name, new Set(pkg.indirectDeps));
+    });
+    /**
+     * Accumulate every package reachable from `info` into its indirectDeps.
+     *
+     * Depth-first over a snapshot of each package's deps, taken as it is reached - the same
+     * order the recursive walk produced. `seen` expands each package once per traversal,
+     * which is all a transitive closure needs and is what keeps a cycle from looping.
+     *
+     * @param info the package whose indirect deps we are accumulating
+     * @returns how many indirect deps were added
+     */
+    const add = (info) => {
+        const seen = new Set();
+        const infoIndirect = indirectDepsSet.get(info.name);
+        const infoLocal = localDepsSet.get(info.name);
+        let added = 0;
+        // each frame is a dep list plus how far into it we have gone
+        const stack = [
+            { deps: info.localDeps.concat(info.indirectDeps), at: 0 },
+        ];
+        while (stack.length > 0) {
+            const frame = stack[stack.length - 1];
+            if (frame.at >= frame.deps.length) {
+                stack.pop();
+                continue;
+            }
+            const dep = frame.deps[frame.at++];
             const depPkg = packages[dep];
-            if (info.localDeps.indexOf(dep) < 0 && info.indirectDeps.indexOf(dep) < 0) {
-                change++;
+            if (!infoLocal.has(dep) && !infoIndirect.has(dep)) {
+                added++;
                 info.indirectDeps.push(dep);
+                infoIndirect.add(dep);
                 depPkg.dependents.push(info.name);
             }
-            if (depPkg.localDeps.indexOf(info.name) >= 0) {
-                const x = [info.name, depPkg.name].sort().join(",");
-                if (circulars.indexOf(x) < 0) {
-                    circulars.push(x);
+            // a cycle that comes straight back to `info` - record the pair and stop descending
+            if (localDepsSet.get(dep).has(info.name)) {
+                const pair = [info.name, depPkg.name].sort().join(",");
+                if (!circularsSet.has(pair)) {
+                    circularsSet.add(pair);
+                    circulars.push(pair);
                 }
-                return;
+                continue;
             }
-            add(info, depPkg.localDeps.concat(depPkg.indirectDeps));
-        });
+            if (seen.has(dep)) {
+                continue;
+            }
+            seen.add(dep);
+            stack.push({ deps: depPkg.localDeps.concat(depPkg.indirectDeps), at: 0 });
+        }
+        return added;
     };
-    lodash_1.default.each(packages, (pkg) => {
-        add(pkg, pkg.localDeps.concat(pkg.indirectDeps));
-    });
-    if (change > 0) {
-        processIndirectDeps(packages, circulars);
-    }
+    // run to a fixpoint: expanding one package can give another package more to reach through
+    let change = 0;
+    do {
+        change = 0;
+        _.each(packages, (pkg) => {
+            change += add(pkg);
+        });
+    } while (change > 0);
 }
 /**
  *
@@ -76,7 +124,7 @@ function processIndirectDeps(packages, circulars) {
  * @param level
  */
 function includeDeps(packages, level) {
-    const localDeps = lodash_1.default.uniq(Object.keys(packages).reduce((acc, p) => {
+    const localDeps = _.uniq(Object.keys(packages).reduce((acc, p) => {
         if (packages[p] && !packages[p].ignore) {
             return acc.concat(packages[p].localDeps.filter((x) => packages[x] && packages[x].ignore));
         }
@@ -97,35 +145,72 @@ function includeDeps(packages, level) {
 /**
  * Read the packages of a fynpo mono-repo
  *
- * @param patterns - array of minimatch patterns.  default: `["packages/*"]`
+ * Honors the same `packages` config as {@link FynpoDepGraph}, so both discovery paths agree.
+ * Passing `patterns` directly still works and takes precedence, for callers that already know
+ * what they want.
+ *
+ * @param patterns - explicit minimatch patterns. Overrides whatever `packages` config says.
+ * @param cwd - repo root
+ * @param packages - raw `packages` config, resolved via {@link resolvePackagesConfig}
  * @returns - packages from the fynpo mono-repo
  */
-async function readFynpoPackages({ patterns = ["packages/*"], cwd = process.cwd(), } = {}) {
-    const mms = patterns.map((p) => new minimatch_1.default.Minimatch(p));
-    const groups = (0, minimatch_group_1.groupMM)(mms, {});
+export async function readFynpoPackages({ patterns = undefined, cwd = process.cwd(), packages = undefined, } = {}) {
+    const config = resolvePackagesConfig(packages);
+    const explicit = _.isEmpty(patterns) ? scanPatterns(config) : patterns;
+    const gitignore = makeGitignoreMatcher(cwd);
+    const excludeMms = config.exclude.map((p) => new Minimatch(p));
+    const isExcluded = (path) => Boolean(path) && excludeMms.some((m) => m.match(path.split(Path.sep).join("/")));
+    // `include` filters what the scan found - it does not replace the scan (FPO-17)
+    const includeMms = (_.isEmpty(patterns) ? includeFilter(config) : []).map((p) => new Minimatch(p));
+    const isIncluded = (path) => includeMms.length === 0 || includeMms.some((m) => m.match(path.split(Path.sep).join("/")));
+    // null patterns means auto-search: scan from the root for every package.json
+    const autoSearch = explicit === null;
+    const groups = autoSearch
+        ? { ".": null }
+        : groupMM(explicit.map((p) => new Minimatch(p)), {});
+    const skipForAutoSearch = (path) => autoSearch && config.autoSearch.respectGitignore && gitignore.ignores(path);
     const files = [];
     for (const prefix in groups) {
-        files.push(await (0, filter_scan_dir_1.filterScanDir)({
+        files.push(await filterScanDir({
             cwd,
             prefix,
             concurrency: 500,
-            filter: (f) => f === "package.json",
-            filterDir: (dir, _p, extras) => {
-                if (dir !== "node_modules") {
-                    return Boolean(groups[prefix].find((save) => save.mm.match(extras.dirFile)));
+            filter: (f, path, extras) => {
+                var _a;
+                if (f !== "package.json") {
+                    return false;
+                }
+                if (autoSearch && (!path || path === ".")) {
+                    // the monorepo's own package.json is not a member
+                    return false;
                 }
-                return false;
+                if (autoSearch && ((_a = extras === null || extras === void 0 ? void 0 : extras.files) === null || _a === void 0 ? void 0 : _a.includes("fynpo.json"))) {
+                    // a nested fynpo root is its own monorepo
+                    return false;
+                }
+                return !isExcluded(path) && !skipForAutoSearch(path);
+            },
+            filterDir: (dir, path, extras) => {
+                if (dir === "node_modules" || isExcluded(path) || skipForAutoSearch(path)) {
+                    return false;
+                }
+                if (autoSearch) {
+                    return !dir.startsWith(".");
+                }
+                return Boolean(groups[prefix].find((save) => save.mm.match(extras.dirFile)));
             },
         }));
     }
-    const allFiles = [].concat(...files).sort();
+    const allFiles = [].concat(...files)
+        .filter((f) => isIncluded(Path.dirname(f)))
+        .sort();
     const allPkgs = {};
     for (const pkgFile of allFiles) {
-        const pkgStr = await fs_1.promises.readFile(path_1.default.join(cwd, pkgFile), "utf-8");
+        const pkgStr = await Fs.readFile(Path.join(cwd, pkgFile), "utf-8");
         const pkgJson = JSON.parse(pkgStr);
-        const path = path_1.default.dirname(pkgFile);
-        const pkgDir = pkgJson.name[0] === "@" && path.endsWith(pkgJson.name) ? pkgJson.name : path_1.default.basename(path);
-        allPkgs[pkgJson.name] = Object.assign(lodash_1.default.pick(pkgJson, [
+        const path = Path.dirname(pkgFile);
+        const pkgDir = pkgJson.name[0] === "@" && path.endsWith(pkgJson.name) ? pkgJson.name : Path.basename(path);
+        allPkgs[pkgJson.name] = Object.assign(_.pick(pkgJson, [
             "name",
             "version",
             "dependencies",
@@ -162,7 +247,7 @@ async function readFynpoPackages({ patterns = ["packages/*"], cwd = process.cwd(
  * @param opts - options
  * @returns
  */
-function makePkgDeps(packages, opts) {
+export function makePkgDeps(packages, opts) {
     const cwd = opts.cwd || process.cwd();
     let circulars = [];
     let ignores = opts.ignore || [];
@@ -180,14 +265,11 @@ function makePkgDeps(packages, opts) {
         }
     }
     // If options.scope is defined, then ignore packages not in it
-    if (opts.scope && opts.scope.length > 0) {
-        Object.keys(packages).forEach((p) => {
-            const scope = p[0] === "@" ? p.slice(0, p.indexOf("/")) : undefined;
-            if ((!scope || !opts.scope.includes(scope)) && !ignores[p]) {
-                ignores.push(p);
-            }
-        });
-    }
+    outOfScopePackages(opts.scope, Object.keys(packages)).forEach((p) => {
+        if (!ignores[p]) {
+            ignores.push(p);
+        }
+    });
     if (opts.only && opts.only.length > 0) {
         opts.only.forEach((x) => {
             if (!packages[x]) {
@@ -200,17 +282,28 @@ function makePkgDeps(packages, opts) {
             }
         });
     }
-    const depMap = lodash_1.default.mapValues(packages, (pkg) => {
-        return lodash_1.default.pick(pkg, ["name", "localDeps", "indirectDeps", "dependents"]);
+    const depMap = _.mapValues(packages, (pkg) => {
+        return _.pick(pkg, ["name", "localDeps", "indirectDeps", "dependents"]);
     });
-    circulars = lodash_1.default.uniq(circulars).map((x) => x.split(","));
-    ignores = ignores.concat(lodash_1.default.map(circulars, (pair) => {
+    circulars = _.uniq(circulars).map((x) => x.split(","));
+    // Breaking a cycle costs a package: the less depended-on half of each pair is dropped from
+    // the run. That used to happen in silence, so a repo with cycles quietly processed fewer
+    // packages than it has and nothing said which ones or why (FPO-43).
+    const circularIgnores = _.uniq(_.map(circulars, (pair) => {
         const depA = packages[pair[0]].dependents.length;
         const depB = packages[pair[1]].dependents.length;
         if (depA === depB)
             return undefined;
         return depA > depB ? pair[1] : pair[0];
     }).filter((x) => x));
+    if (circulars.length > 0) {
+        warnings.push(`Circular local dependencies: ${circulars.map((pair) => pair.join(" <-> ")).join(", ")}`);
+        if (circularIgnores.length > 0) {
+            warnings.push(`Ignoring ${circularIgnores.join(", ")} to break the circular dependencies above - ` +
+                `they are dropped from this run. Break the cycles to include them.`);
+        }
+    }
+    ignores = ignores.concat(circularIgnores);
     ignores.forEach((x) => {
         if (packages[x]) {
             packages[x].ignore = true;
@@ -231,4 +324,3 @@ function makePkgDeps(packages, opts) {
         focusPkgPath,
     };
 }
-//# sourceMappingURL=index.js.map

package/dist/minimatch-group.d.ts

@@ -1,12 +1,12 @@
-import mm from "minimatch";
+import { Minimatch, type ParseReturnFiltered } from "minimatch";
 /**
  * Remember the minimatch group info from a pattern
  */
 type MMGroup = {
     /** the minimatch object */
-    mm: mm.IMinimatch;
-    /** the minimatch sets */
-    set: any[][];
+    mm: Minimatch;
+    /** one expanded pattern set (a row of Minimatch.set) */
+    set: ParseReturnFiltered[];
     /** index of the set within the minimatch object */
     setIx: number;
     /** index of the first set that's not a literal string */
@@ -22,16 +22,16 @@ export type MMGroups = Record<string, MMGroup[]>;
  * @param groups - object to group the minimatch objects
  * @returns object of grouped minimatch objects
  */
-export declare function groupMM(mms: mm.IMinimatch[], groups: MMGroups): MMGroups;
+export declare function groupMM(mms: Minimatch[], groups: MMGroups): MMGroups;
 /**
  * process a minimatch pattern to group them for matching directories
  *
  * - needs to create a new pattern at every non-string part
  * @param m0 minimatch pattern
  */
-export declare function deconstructMM(m0: mm.IMinimatch): {
-    m0: mm.IMinimatch;
-    mms: mm.IMinimatch[];
+export declare function deconstructMM(m0: Minimatch): {
+    m0: Minimatch;
+    mms: Minimatch[];
 };
 /**
  * check that a path matches against a list of minimatch patterns
@@ -40,7 +40,7 @@ export declare function deconstructMM(m0: mm.IMinimatch): {
  * @param patterns
  * @returns the first pattern that match or false
  */
-export declare function checkMmMatch(fullPath: string, patterns: mm.IMinimatch[]): false | mm.IMinimatch;
+export declare function checkMmMatch(fullPath: string, patterns: Minimatch[]): false | Minimatch;
 /**
  * Take a full path and match each level of it to a list of minimatch patterns
  *
@@ -48,5 +48,5 @@ export declare function checkMmMatch(fullPath: string, patterns: mm.IMinimatch[]
  * @param mms
  * @returns
  */
-export declare function unrollMmMatch(path: string, mms: mm.IMinimatch[]): false | mm.IMinimatch;
+export declare function unrollMmMatch(path: string, mms: Minimatch[]): false | Minimatch;
 export {};

package/dist/minimatch-group.js

@@ -1,12 +1,5 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.groupMM = groupMM;
-exports.deconstructMM = deconstructMM;
-exports.checkMmMatch = checkMmMatch;
-exports.unrollMmMatch = unrollMmMatch;
-const tslib_1 = require("tslib");
-const minimatch_1 = tslib_1.__importDefault(require("minimatch"));
-const lodash_1 = tslib_1.__importDefault(require("lodash"));
+import { Minimatch, GLOBSTAR } from "minimatch";
+import _ from "lodash";
 /**
  * process a list of minimatch objects and group them by the string prefix of their patterns
  *
@@ -14,7 +7,7 @@ const lodash_1 = tslib_1.__importDefault(require("lodash"));
  * @param groups - object to group the minimatch objects
  * @returns object of grouped minimatch objects
  */
-function groupMM(mms, groups) {
+export function groupMM(mms, groups) {
     mms.forEach((mm) => {
         mm.set.forEach((set, setIx) => {
             const ix = set.findIndex((s) => typeof s !== "string");
@@ -43,20 +36,19 @@ function groupMM(mms, groups) {
  * - needs to create a new pattern at every non-string part
  * @param m0 minimatch pattern
  */
-function deconstructMM(m0) {
+export function deconstructMM(m0) {
     const mms = [];
     const patterns = { m0, mms };
     const set = m0.set[0];
     const globParts = m0.globParts[0];
     const iParts = [];
-    const { GLOBSTAR } = minimatch_1.default;
     for (let ix = 0; ix < set.length; ix++) {
         const s = set[ix];
         const g = globParts[ix];
         // if we hit something that's not string, then we need a mm with just the strings
         // because a dir like "src" will not match "src/*"
         if (typeof s !== "string" && iParts.length > 0) {
-            mms.push(new minimatch_1.default.Minimatch(iParts.join("/"), m0.options));
+            mms.push(new Minimatch(iParts.join("/"), m0.options));
         }
         if (ix === set.length - 1) {
             mms.push(m0);
@@ -65,7 +57,7 @@ function deconstructMM(m0) {
         else {
             iParts.push(g);
             if (typeof s !== "string") {
-                mms.push(new minimatch_1.default.Minimatch(iParts.join("/"), m0.options));
+                mms.push(new Minimatch(iParts.join("/"), m0.options));
             }
         }
         if (s === GLOBSTAR) {
@@ -82,8 +74,14 @@ function deconstructMM(m0) {
  * @param patterns
  * @returns the first pattern that match or false
  */
-function checkMmMatch(fullPath, patterns) {
-    return !lodash_1.default.isEmpty(patterns) && patterns.find((patternMm) => patternMm.match(fullPath));
+export function checkMmMatch(fullPath, patterns) {
+    // Callers append a trailing "/" to force matching a directory.  minimatch 3 never let a
+    // pattern part that can match the empty string (e.g. "?(a|b)") consume that trailing empty
+    // segment, but minimatch 10 does, so "**/?(a|b)" would wrongly match "src/".  For every
+    // other case v3's match of "dir/" is equivalent to matching "dir", so strip the trailing
+    // slash to keep the v3 semantics.
+    const path = fullPath.length > 1 && fullPath.endsWith("/") ? fullPath.slice(0, -1) : fullPath;
+    return !_.isEmpty(patterns) && patterns.find((patternMm) => patternMm.match(path));
 }
 /**
  * Take a full path and match each level of it to a list of minimatch patterns
@@ -92,7 +90,7 @@ function checkMmMatch(fullPath, patterns) {
  * @param mms
  * @returns
  */
-function unrollMmMatch(path, mms) {
+export function unrollMmMatch(path, mms) {
     const parts = path.split("/");
     let rp;
     for (let i = 0; i < parts.length - 1; i++) {
@@ -104,4 +102,3 @@ function unrollMmMatch(path, mms) {
     }
     return checkMmMatch(path, mms);
 }
-//# sourceMappingURL=minimatch-group.js.map

package/dist/packages-config.d.ts

@@ -0,0 +1,78 @@
+/** auto-search settings, after defaults are applied */
+export type AutoSearchConfig = {
+    /** search the whole repo for package.json when no explicit `include` patterns are given */
+    enable: boolean;
+    /** when true, auto-search skips gitignored paths. Does NOT affect the publish veto. */
+    respectGitignore: boolean;
+};
+/** the `packages` config, after defaults are applied */
+export type PackagesConfig = {
+    autoSearch: AutoSearchConfig;
+    /** explicit discovery patterns. Empty means "not specified" */
+    include: string[];
+    /** applies to every package, auto-searched or explicitly matched */
+    exclude: string[];
+    /** publish allow list. Empty means every discovered package is eligible */
+    publishInclude: string[];
+    /** publish deny list, applied after the allow list */
+    publishExclude: string[];
+};
+/**
+ * Normalize `packages` from a fynpo config into a complete {@link PackagesConfig}.
+ *
+ * Accepts both shapes:
+ *
+ * - **array** - the historical form. Treated as `publishInclude`, with auto-search on and
+ *   `respectGitignore` off. It no longer narrows discovery.
+ * - **object** - `{ autoSearch, include, exclude, publishInclude, publishExclude }`.
+ *
+ * Defaults: `autoSearch` on, `respectGitignore` off. With auto-search off and no `include`,
+ * `include` falls back to `["packages/*"]`.
+ *
+ * @param packages - the raw `packages` value from fynpo.json / fynpo.config.js
+ * @returns the resolved config, every field populated
+ */
+export declare function resolvePackagesConfig(packages?: unknown): PackagesConfig;
+/**
+ * The npm scope of a package name, or undefined when it is unscoped.
+ *
+ * @param name - package name
+ * @returns the scope including its `@`, e.g. `@fynjs`
+ */
+export declare function packageScope(name: string): string | undefined;
+/**
+ * Package names that fall OUTSIDE the given scopes, i.e. the ones `--scope` should exclude.
+ *
+ * An unscoped package is never in any scope, so it is always excluded once `--scope` is given.
+ * Scopes may be written with or without the leading `@`.
+ *
+ * Shared by both selection paths so they cannot disagree: `makePkgDeps` (used by prepare) and
+ * the dep-graph path (used by bootstrap, local and run). `--scope` used to be applied only in
+ * the former, so the three commands that actually advertise it silently ignored it. See FPO-37.
+ *
+ * @param scopes - the requested scopes
+ * @param names - all known package names
+ * @returns names to exclude; empty when no scope was requested
+ */
+export declare function outOfScopePackages(scopes: unknown, names: string[]): string[];
+/**
+ * Decide how to scan for packages.
+ *
+ * `include` does NOT turn auto-search off - auto-search is on by default and stays on, so it
+ * still decides *how the tree is walked*. `include` then filters what the walk found, via
+ * {@link includeFilter}. Only with auto-search off does `include` become the scan patterns
+ * themselves, falling back to `packages/*`.
+ *
+ * @param config - resolved packages config
+ * @returns `null` to auto-search the whole repo, otherwise the patterns to scan
+ */
+export declare function scanPatterns(config: PackagesConfig): string[] | null;
+/**
+ * The patterns a discovered package must match to be kept.
+ *
+ * Empty means keep everything the scan found.
+ *
+ * @param config - resolved packages config
+ * @returns patterns to match a package path against
+ */
+export declare function includeFilter(config: PackagesConfig): string[];