@cdxgen/cdxgen 0.0.1 → 13.0.0

package/lib/third-party/arborist/lib/arborist/load-actual.js

@@ -0,0 +1,605 @@
+import { dirname, join, normalize, relative, resolve, sep } from "node:path";
+
+import { readdirScoped } from "@npmcli/fs";
+import PackageJson from "@npmcli/package-json";
+import { commonAncestorPath } from "common-ancestor-path";
+import { walkUp } from "walk-up-path";
+
+import calcDepFlags from "../calc-dep-flags.js";
+import Link from "../link.js";
+import Node from "../node.js";
+import NpmExtension from "../npm-extension.js";
+import PackageExtensions from "../package-extensions.js";
+import realpath from "../realpath.js";
+import Shrinkwrap from "../shrinkwrap.js";
+import treeCheck from "../tree-check.js";
+
+// mix-in implementing the loadActual method
+
+// public symbols
+const _changePath = Symbol.for("_changePath");
+const _setWorkspaces = Symbol.for("setWorkspaces");
+const _rpcache = Symbol.for("realpathCache");
+const _stcache = Symbol.for("statCache");
+
+/** @returns {any} */
+export default (cls) =>
+  class ActualLoader extends cls {
+    constructor(options) {
+      super(options);
+      const cwd = process.cwd();
+      this[_rpcache] = new Map([[cwd, cwd]]);
+      this[_stcache] = new Map();
+    }
+
+    #actualTree;
+    // ensure when walking the tree that we don't call loadTree on the same
+    // actual node more than one time.
+    #actualTreeLoaded = new Set();
+    #actualTreePromise;
+
+    // cache of nodes when loading the actualTree, so that we avoid loaded the
+    // same node multiple times when symlinks attack.
+    #cache = new Map();
+    #filter;
+
+    // cache of link targets for setting fsParent links
+    // We don't do fsParent as a magic getter/setter, because it'd be too costly
+    // to keep up to date along the walk.
+    // And, we know that it can ONLY be relevant when the node is a target of a
+    // link; otherwise, it'd be in a node_modules folder, so take advantage of
+    // that to limit the scans later.
+    #topNodes = new Set();
+    #transplantFilter;
+
+    // public method
+    // TODO remove options param in next semver major
+    async loadActual(options = {}) {
+      // In the past this.actualTree was set as a promise that eventually
+      // resolved, and overwrite this.actualTree with the resolved value.  This
+      // was a problem because virtually no other code expects this.actualTree to
+      // be a promise.  Instead we only set it once resolved, and also return it
+      // from the promise so that it is what's returned from this function when
+      // awaited.
+      if (this.actualTree) {
+        return this.actualTree;
+      }
+      if (!this.#actualTreePromise) {
+        // allow the user to set options on the ctor as well.
+        // XXX: deprecate separate method options objects.
+        options = { ...this.options, ...options };
+
+        this.#actualTreePromise = this.#loadActual(options).then((tree) => {
+          // reset all deps to extraneous prior to recalc
+          if (!options.root) {
+            for (const node of tree.inventory.values()) {
+              node.extraneous = true;
+            }
+          }
+
+          // only reset root flags if we're not re-rooting,
+          // otherwise leave as-is
+          calcDepFlags(tree, !options.root);
+          this.#repropagateOverrides();
+          this.actualTree = treeCheck(tree);
+          return this.actualTree;
+        });
+      }
+      return this.#actualTreePromise;
+    }
+
+    // return the promise so that we don't ever have more than one going at the
+    // same time.  This is so that buildIdealTree can default to the actualTree
+    // if no shrinkwrap present, but reify() can still call buildIdealTree and
+    // loadActual in parallel safely.
+
+    async #loadActual(options) {
+      // mostly realpath to throw if the root doesn't exist
+      const {
+        global,
+        filter = () => true,
+        root = null,
+        transplantFilter = () => true,
+        ignoreMissing = false,
+        forceActual = false,
+      } = options;
+      this.#filter = filter;
+      this.#transplantFilter = transplantFilter;
+
+      if (global) {
+        const real = await realpath(this.path, this[_rpcache], this[_stcache]);
+        const params = {
+          path: this.path,
+          realpath: real,
+          pkg: {},
+          global,
+          loadOverrides: true,
+        };
+        if (this.path === real) {
+          this.#actualTree = this.#newNode(params);
+        } else {
+          this.#actualTree = await this.#newLink(params);
+        }
+      } else {
+        // not in global mode, hidden lockfile is allowed, load root pkg too
+        this.#actualTree = await this.#loadFSNode({
+          path: this.path,
+          real: await realpath(this.path, this[_rpcache], this[_stcache]),
+          loadOverrides: true,
+        });
+
+        this.#actualTree.assertRootOverrides();
+
+        // if forceActual is set, don't even try the hidden lockfile
+        if (!forceActual) {
+          // Note: hidden lockfile will be rejected if it's not the latest thing
+          // in the folder, or if any of the entries in the hidden lockfile are
+          // missing.
+          const meta = await Shrinkwrap.load({
+            path: this.#actualTree.path,
+            hiddenLockfile: true,
+            resolveOptions: this.options,
+          });
+
+          if (meta.loadedFromDisk) {
+            this.#actualTree.meta = meta;
+            // have to load on a new Arborist object, so we don't assign
+            // the virtualTree on this one!  Also, the weird reference is because
+            // we can't easily get a ref to Arborist in this module, without
+            // creating a circular reference, since this class is a mixin used
+            // to build up the Arborist class itself.
+            await new this.constructor({ ...this.options }).loadVirtual({
+              root: this.#actualTree,
+            });
+            await this[_setWorkspaces](this.#actualTree);
+            this.#linkActualWorkspaces();
+
+            this.#transplant(root);
+            return this.#actualTree;
+          }
+        }
+
+        const meta = await Shrinkwrap.load({
+          path: this.#actualTree.path,
+          lockfileVersion: this.options.lockfileVersion,
+          resolveOptions: this.options,
+        });
+        this.#actualTree.meta = meta;
+      }
+
+      await this.#loadFSTree(this.#actualTree);
+      await this[_setWorkspaces](this.#actualTree);
+
+      // if there are workspace targets without Link nodes created, load
+      // the targets, so that we know what they are.
+      if (this.#actualTree.workspaces && this.#actualTree.workspaces.size) {
+        const promises = [];
+        for (const path of this.#actualTree.workspaces.values()) {
+          if (!this.#cache.has(path)) {
+            // workspace overrides use the root overrides
+            const p = this.#loadFSNode({
+              path,
+              root: this.#actualTree,
+              useRootOverrides: true,
+            }).then((node) => this.#loadFSTree(node));
+            promises.push(p);
+          }
+        }
+        await Promise.all(promises);
+      }
+
+      this.#linkActualWorkspaces();
+
+      // .npm-extension runs before packageExtensions, matching the ideal-tree resolution order
+      await this.#applyNpmExtension();
+      this.#applyPackageExtensions();
+
+      if (!ignoreMissing) {
+        await this.#findMissingEdges();
+      }
+
+      // try to find a node that is the parent in a fs tree sense, but not a
+      // node_modules tree sense, of any link targets.  this allows us to
+      // resolve deps that node will find, but a legacy npm view of the
+      // world would not have noticed.
+      for (const path of this.#topNodes) {
+        const node = this.#cache.get(path);
+        if (node && !node.parent && !node.fsParent) {
+          for (const p of walkUp(dirname(path))) {
+            if (this.#cache.has(p)) {
+              node.fsParent = this.#cache.get(p);
+              break;
+            }
+          }
+        }
+      }
+
+      this.#transplant(root);
+
+      if (global) {
+        // need to depend on the children, or else all of them
+        // will end up being flagged as extraneous, since the
+        // global root isn't a "real" project
+        const tree = this.#actualTree;
+        const actualRoot = tree.isLink ? tree.target : tree;
+        const { dependencies = {} } = actualRoot.package;
+        for (const [name, kid] of actualRoot.children.entries()) {
+          const def = kid.isLink ? `file:${kid.realpath}` : "*";
+          dependencies[name] = dependencies[name] || def;
+        }
+        actualRoot.package = { ...actualRoot.package, dependencies };
+      }
+      return this.#actualTree;
+    }
+
+    // Under the linked (isolated) strategy, workspaces the root does not depend on are not symlinked into the root node_modules, so neither the on-disk scan nor the hidden lockfile gives the root's workspace edges a node_modules/<ws> link to resolve to.
+    // Synthesize those links from the authoritative workspaces map so npm ls and npm query surface the workspaces, matching hoisted and the logical package-lock.
+    #linkActualWorkspaces() {
+      const root = this.#actualTree;
+      if (this.options.installStrategy !== "linked" || !root.workspaces) {
+        return;
+      }
+      // Declared workspaces ARE symlinked at root node_modules under linked, so a missing link there is a real problem that must surface as UNMET.
+      // Only undeclared workspaces are intentionally absent from root node_modules and need a synthesized link so their root edges resolve.
+      const pkg = root.package;
+      const declared = new Set(
+        Object.keys(
+          Object.assign(
+            {},
+            pkg.dependencies,
+            pkg.devDependencies,
+            pkg.optionalDependencies,
+            pkg.peerDependencies,
+          ),
+        ),
+      );
+      // index loaded workspace targets by both path and realpath, so a workspace reached through a symlinked dir still matches
+      const byLoc = new Map();
+      for (const node of root.fsChildren) {
+        byLoc.set(node.path, node);
+        byLoc.set(node.realpath, node);
+      }
+      for (const [name, path] of root.workspaces.entries()) {
+        // declared workspaces, and any name already a root child, resolve their edge without a synthesized link
+        if (declared.has(name) || root.children.has(name)) {
+          continue;
+        }
+        const target = byLoc.get(path);
+        if (target) {
+          // eslint-disable-next-line no-new
+          new Link({
+            name,
+            path: resolve(root.path, "node_modules", name),
+            realpath: target.realpath,
+            target,
+            parent: root,
+            root,
+          });
+        }
+      }
+    }
+
+    #transplant(root) {
+      if (!root || root === this.#actualTree) {
+        return;
+      }
+
+      this.#actualTree[_changePath](root.path);
+      for (const node of this.#actualTree.children.values()) {
+        if (!this.#transplantFilter(node)) {
+          node.root = null;
+        }
+      }
+
+      root.replace(this.#actualTree);
+      for (const node of this.#actualTree.fsChildren) {
+        node.root = this.#transplantFilter(node) ? root : null;
+      }
+
+      this.#actualTree = root;
+    }
+
+    async #loadFSNode({
+      path,
+      parent,
+      real,
+      root,
+      loadOverrides,
+      useRootOverrides,
+    }) {
+      if (!real) {
+        try {
+          real = await realpath(path, this[_rpcache], this[_stcache]);
+        } catch (error) {
+          // if realpath fails, just provide a dummy error node
+          return new Node({
+            error,
+            path,
+            realpath: path,
+            parent,
+            root,
+            loadOverrides,
+          });
+        }
+      }
+
+      const cached = this.#cache.get(path);
+      let node;
+      // missing edges get a dummy node, assign the parent and return it
+      if (cached && !cached.dummy) {
+        cached.parent = parent;
+        return cached;
+      } else {
+        const params = {
+          installLinks: this.installLinks,
+          legacyPeerDeps: this.legacyPeerDeps,
+          path,
+          realpath: real,
+          parent,
+          root,
+          loadOverrides,
+          // A package physically located in the linked strategy's store is a transitive dependency, not a real tree top, so it must not load its devDependencies.
+          // Never flag the loaded root itself, even if its own path happens to sit under a .store directory.
+          isInStore:
+            path !== this.path &&
+            real.includes(`${sep}node_modules${sep}.store${sep}`),
+        };
+
+        try {
+          const { content: pkg } = await PackageJson.normalize(real);
+          params.pkg = pkg;
+          if (useRootOverrides && root.overrides) {
+            params.overrides = root.overrides.getNodeRule({
+              name: pkg.name,
+              version: pkg.version,
+            });
+          }
+        } catch (err) {
+          if (err.code === "EJSONPARSE") {
+            // TODO @npmcli/package-json should be doing this
+            err.path = join(real, "package.json");
+          }
+          params.error = err;
+        }
+
+        // soldier on if read-package-json raises an error, passing it to the
+        // Node which will attach it to its errors array (Link passes it along to
+        // its target node)
+        if (normalize(path) === real) {
+          node = this.#newNode(params);
+        } else {
+          node = await this.#newLink(params);
+        }
+      }
+      this.#cache.set(path, node);
+      return node;
+    }
+
+    #newNode(options) {
+      // check it for an fsParent if it's a tree top.  there's a decent chance
+      // it'll get parented later, making the fsParent scan a no-op, but better
+      // safe than sorry, since it's cheap.
+      const { parent, realpath } = options;
+      if (!parent) {
+        this.#topNodes.add(realpath);
+      }
+      return new Node(options);
+    }
+
+    async #newLink(options) {
+      const { realpath } = options;
+      this.#topNodes.add(realpath);
+      const target = this.#cache.get(realpath);
+      const link = new Link({ ...options, target });
+
+      if (!target) {
+        // Link set its target itself in this case
+        this.#cache.set(realpath, link.target);
+        // if a link target points at a node outside of the root tree's
+        // node_modules hierarchy, then load that node as well.
+        await this.#loadFSTree(link.target);
+      }
+
+      return link;
+    }
+
+    async #loadFSTree(node) {
+      const did = this.#actualTreeLoaded;
+      if (!node.isLink && !did.has(node.target.realpath)) {
+        did.add(node.target.realpath);
+        await this.#loadFSChildren(node.target);
+        return Promise.all(
+          [...node.target.children.entries()]
+            .filter(([, kid]) => !did.has(kid.realpath))
+            .map(([, kid]) => this.#loadFSTree(kid)),
+        );
+      }
+    }
+
+    // create child nodes for all the entries in node_modules
+    // and attach them to the node as a parent
+    async #loadFSChildren(node) {
+      const nm = resolve(node.realpath, "node_modules");
+      try {
+        const kids = await readdirScoped(nm).then((paths) =>
+          paths.map((p) => p.replace(/\\/g, "/")),
+        );
+        return Promise.all(
+          // ignore . dirs and retired scoped package folders
+          kids
+            .filter((kid) => !/^(@[^/]+\/)?\./.test(kid))
+            .filter((kid) => this.#filter(node, kid))
+            .map((kid) =>
+              this.#loadFSNode({
+                parent: node,
+                path: resolve(nm, kid),
+              }),
+            ),
+        );
+      } catch {
+        // error in the readdir is not fatal, just means no kids
+      }
+    }
+
+    // packageExtensions never rewrite a package's package.json, so a filesystem-scanned actual tree lacks the extension-created edges and provenance.
+    // Re-derive them from the root rule set, as buildIdealTree does.
+    // This is always required under the linked strategy, whose store layout forces the filesystem-scan path.
+    #applyPackageExtensions() {
+      const rootPkg = this.#actualTree.target?.package;
+      const pe = new PackageExtensions(rootPkg?.packageExtensions);
+      if (!pe.present || !pe.selectors.length) {
+        return;
+      }
+      for (const node of this.#actualTree.inventory.values()) {
+        // only installed dependencies are extended, never the root or a workspace
+        if (
+          node.isLink ||
+          node.isProjectRoot ||
+          !node.name ||
+          !node.inNodeModules()
+        ) {
+          continue;
+        }
+        const res = pe.apply(node.package);
+        if (res) {
+          node.package = res.pkg;
+          node.packageExtensionsApplied = res.applied;
+        }
+      }
+      // mirror the provenance onto links so the logical tree location reports it too
+      for (const node of this.#actualTree.inventory.values()) {
+        if (node.isLink && node.target?.packageExtensionsApplied) {
+          node.packageExtensionsApplied = node.target.packageExtensionsApplied;
+        }
+      }
+    }
+
+    // Re-forward overrides through links once all edges are resolved, since a Link may forward before its subtree resolves and miss a transitive match (npm/cli#9619, #9659).
+    #repropagateOverrides() {
+      if (!this.#actualTree.overrides) {
+        return;
+      }
+      for (const node of this.#actualTree.inventory.values()) {
+        if (node.isLink && node.overrides) {
+          node.recalculateOutEdgesOverrides();
+        }
+      }
+    }
+
+    // .npm-extension transformManifest, like packageExtensions, never rewrites a package's package.json, so re-derive its edges and provenance on a filesystem-scanned actual tree.
+    // This executes the root extension code; ignore-extension (and ignore-scripts via flatten) disables it.
+    async #applyNpmExtension() {
+      if (this.options.ignoreExtension) {
+        return;
+      }
+      const ext = new NpmExtension({
+        root: this.#actualTree.realpath,
+        extensionFile: this.options.extensionFile,
+      });
+      if (!ext.present) {
+        return;
+      }
+      await ext.load();
+      for (const node of this.#actualTree.inventory.values()) {
+        // only installed dependencies are transformed, never the root or a workspace
+        if (
+          node.isLink ||
+          node.isProjectRoot ||
+          !node.name ||
+          !node.inNodeModules()
+        ) {
+          continue;
+        }
+        const res = ext.apply(node.package);
+        if (res) {
+          node.package = res.pkg;
+          node.npmExtensionApplied = res.applied;
+        }
+      }
+      // mirror the provenance onto links so the logical tree location reports it too
+      for (const node of this.#actualTree.inventory.values()) {
+        if (node.isLink && node.target?.npmExtensionApplied) {
+          node.npmExtensionApplied = node.target.npmExtensionApplied;
+        }
+      }
+    }
+
+    async #findMissingEdges() {
+      // try to resolve any missing edges by walking up the directory tree,
+      // checking for the package in each node_modules folder.  stop at the
+      // root directory.
+      // The tricky move here is that we load a "dummy" node for the folder
+      // containing the node_modules folder, so that it can be assigned as
+      // the fsParent.  It's a bad idea to *actually* load that full node,
+      // because people sometimes develop in ~/projects/node_modules/...
+      // so we'd end up loading a massive tree with lots of unrelated junk.
+      const nmContents = new Map();
+      const tree = this.#actualTree;
+      for (const node of tree.inventory.values()) {
+        const ancestor = commonAncestorPath(node.realpath, this.path);
+
+        const depPromises = [];
+        for (const [name, edge] of node.edgesOut.entries()) {
+          // An unresolved optional edge reports missing === false, so check the target directly.
+          // Otherwise an installed optional dep that lives only as a store sibling is never loaded.
+          const resolved = edge.to && !edge.to.dummy && edge.to.parent === node;
+          if (resolved) {
+            continue;
+          }
+
+          // start the walk from the dirname, because we would have found
+          // the dep in the loadFSTree step already if it was local.
+          for (const p of walkUp(dirname(node.realpath))) {
+            // only walk as far as the nearest ancestor
+            // this keeps us from going into completely unrelated
+            // places when a project is just missing something, but
+            // allows for finding the transitive deps of link targets.
+            // ie, if it has to go up and back out to get to the path
+            // from the nearest common ancestor, we've gone too far.
+            if (ancestor && /^\.\.(?:[\\/]|$)/.test(relative(ancestor, p))) {
+              break;
+            }
+
+            let entries;
+            if (!nmContents.has(p)) {
+              entries = await readdirScoped(p + "/node_modules")
+                .catch(() => [])
+                .then((paths) => paths.map((p) => p.replace(/\\/g, "/")));
+              nmContents.set(p, entries);
+            } else {
+              entries = nmContents.get(p);
+            }
+
+            if (!entries.includes(name)) {
+              continue;
+            }
+
+            let d;
+            if (!this.#cache.has(p)) {
+              d = new Node({ path: p, root: node.root, dummy: true });
+              this.#cache.set(p, d);
+            } else {
+              d = this.#cache.get(p);
+            }
+            if (d.dummy) {
+              // it's a placeholder, so likely would not have loaded this dep,
+              // unless another dep in the tree also needs it.
+              const depPath = normalize(`${p}/node_modules/${name}`);
+              const cached = this.#cache.get(depPath);
+              if (!cached || cached.dummy) {
+                depPromises.push(
+                  this.#loadFSNode({
+                    path: depPath,
+                    root: node.root,
+                    parent: d,
+                  }).then((node) => this.#loadFSTree(node)),
+                );
+              }
+            }
+            break;
+          }
+        }
+        await Promise.all(depPromises);
+      }
+    }
+  };