@npmcli/arborist 9.0.1 → 9.0.2

package/lib/arborist/build-ideal-tree.js

@@ -13,6 +13,7 @@ const { lstat, readlink } = require('node:fs/promises')
 const { depth } = require('treeverse')
 const { log, time } = require('proc-log')
 const { redact } = require('@npmcli/redact')
+const semver = require('semver')
 
 const {
   OK,
@@ -279,14 +280,23 @@ module.exports = cls => class IdealTreeBuilder extends cls {
       // When updating all, we load the shrinkwrap, but don't bother
       // to build out the full virtual tree from it, since we'll be
       // reconstructing it anyway.
-      .then(root => this.options.global ? root
-      : !this[_usePackageLock] || this[_updateAll]
-        ? Shrinkwrap.reset({
-          path: this.path,
-          lockfileVersion: this.options.lockfileVersion,
-          resolveOptions: this.options,
-        }).then(meta => Object.assign(root, { meta }))
-        : this.loadVirtual({ root }))
+      .then(root => {
+        if (this.options.global) {
+          return root
+        } else if (!this[_usePackageLock] || this[_updateAll]) {
+          return Shrinkwrap.reset({
+            path: this.path,
+            lockfileVersion: this.options.lockfileVersion,
+            resolveOptions: this.options,
+          }).then(meta => Object.assign(root, { meta }))
+        } else {
+          return this.loadVirtual({ root })
+            .then(tree => {
+              this.#applyRootOverridesToWorkspaces(tree)
+              return tree
+            })
+        }
+      })
 
       // if we don't have a lockfile to go from, then start with the
       // actual tree, so we only make the minimum required changes.
@@ -799,7 +809,8 @@ This is a one-time fix-up, please be patient...
     const crackOpen = this.#complete &&
       node !== this.idealTree &&
       node.resolved &&
-      (hasBundle || hasShrinkwrap)
+      (hasBundle || hasShrinkwrap) &&
+      !node.ideallyInert
     if (crackOpen) {
       const Arborist = this.constructor
       const opt = { ...this.options }
@@ -1475,6 +1486,32 @@ This is a one-time fix-up, please be patient...
     timeEnd()
   }
 
+  #applyRootOverridesToWorkspaces (tree) {
+    const rootOverrides = tree.root.package.overrides || {}
+
+    for (const node of tree.root.inventory.values()) {
+      if (!node.isWorkspace) {
+        continue
+      }
+
+      for (const depName of Object.keys(rootOverrides)) {
+        const edge = node.edgesOut.get(depName)
+        const rootNode = tree.root.children.get(depName)
+
+        // safely skip if either edge or rootNode doesn't exist yet
+        if (!edge || !rootNode) {
+          continue
+        }
+
+        const resolvedRootVersion = rootNode.package.version
+        if (!semver.satisfies(resolvedRootVersion, edge.spec)) {
+          edge.detach()
+          node.children.delete(depName)
+        }
+      }
+    }
+  }
+
   #idealTreePrune () {
     for (const node of this.idealTree.inventory.values()) {
       if (node.extraneous) {
@@ -1491,7 +1528,7 @@ This is a one-time fix-up, please be patient...
 
       const set = optionalSet(node)
       for (const node of set) {
-        node.parent = null
+        node.ideallyInert = true
       }
     }
   }
@@ -1512,6 +1549,7 @@ This is a one-time fix-up, please be patient...
           node.parent !== null
           && !node.isProjectRoot
           && !excludeNodes.has(node)
+          && !node.ideallyInert
         ) {
           this[_addNodeToTrashList](node)
         }

package/lib/arborist/isolated-reifier.js

@@ -81,7 +81,7 @@ module.exports = cls => class IsolatedReifier extends cls {
         }
         queue.push(e.to)
       })
-      if (!next.isProjectRoot && !next.isWorkspace) {
+      if (!next.isProjectRoot && !next.isWorkspace && !next.ideallyInert) {
         root.external.push(await this.externalProxyMemo(next))
       }
     }
@@ -147,8 +147,8 @@ module.exports = cls => class IsolatedReifier extends cls {
     const nonOptionalDeps = edges.filter(e => !e.optional).map(e => e.to.target)
 
     result.localDependencies = await Promise.all(nonOptionalDeps.filter(n => n.isWorkspace).map(this.workspaceProxyMemo))
-    result.externalDependencies = await Promise.all(nonOptionalDeps.filter(n => !n.isWorkspace).map(this.externalProxyMemo))
-    result.externalOptionalDependencies = await Promise.all(optionalDeps.map(this.externalProxyMemo))
+    result.externalDependencies = await Promise.all(nonOptionalDeps.filter(n => !n.isWorkspace && !n.ideallyInert).map(this.externalProxyMemo))
+    result.externalOptionalDependencies = await Promise.all(optionalDeps.filter(n => !n.ideallyInert).map(this.externalProxyMemo))
     result.dependencies = [
       ...result.externalDependencies,
       ...result.localDependencies,

package/lib/arborist/load-virtual.js

@@ -267,6 +267,7 @@ module.exports = cls => class VirtualLoader extends cls {
       integrity: sw.integrity,
       resolved: consistentResolve(sw.resolved, this.path, path),
       pkg: sw,
+      ideallyInert: sw.ideallyInert,
       hasShrinkwrap: sw.hasShrinkwrap,
       dev,
       optional,

package/lib/arborist/reify.js

@@ -8,7 +8,6 @@ const semver = require('semver')
 const debug = require('../debug.js')
 const { walkUp } = require('walk-up-path')
 const { log, time } = require('proc-log')
-const hgi = require('hosted-git-info')
 const rpj = require('read-package-json-fast')
 
 const { dirname, resolve, relative, join } = require('node:path')
@@ -150,7 +149,7 @@ module.exports = cls => class Reifier extends cls {
     for (const path of this[_trashList]) {
       const loc = relpath(this.idealTree.realpath, path)
       const node = this.idealTree.inventory.get(loc)
-      if (node && node.root === this.idealTree) {
+      if (node && node.root === this.idealTree && !node.ideallyInert) {
         node.parent = null
       }
     }
@@ -238,6 +237,18 @@ module.exports = cls => class Reifier extends cls {
     this.idealTree.meta.hiddenLockfile = true
     this.idealTree.meta.lockfileVersion = defaultLockfileVersion
 
+    // Preserve inertness for failed stuff.
+    if (this.actualTree) {
+      for (const [loc, actual] of this.actualTree.inventory.entries()) {
+        if (actual.ideallyInert) {
+          const ideal = this.idealTree.inventory.get(loc)
+          if (ideal) {
+            ideal.ideallyInert = true
+          }
+        }
+      }
+    }
+
     this.actualTree = this.idealTree
     this.idealTree = null
 
@@ -600,6 +611,9 @@ module.exports = cls => class Reifier extends cls {
     // retire the same path at the same time.
     const dirsChecked = new Set()
     return promiseAllRejectLate(leaves.map(async node => {
+      if (node.ideallyInert) {
+        return
+      }
       for (const d of walkUp(node.path)) {
         if (d === node.top.path) {
           break
@@ -744,6 +758,10 @@ module.exports = cls => class Reifier extends cls {
   }
 
   async #extractOrLink (node) {
+    if (node.ideallyInert) {
+      return
+    }
+
     const nm = resolve(node.parent.path, 'node_modules')
     await this.#validateNodeModules(nm)
 
@@ -819,6 +837,7 @@ module.exports = cls => class Reifier extends cls {
       const set = optionalSet(node)
       for (node of set) {
         log.verbose('reify', 'failed optional dependency', node.path)
+        node.ideallyInert = true
         this[_addNodeToTrashList](node)
       }
     }) : p).then(() => node)
@@ -833,21 +852,23 @@ module.exports = cls => class Reifier extends cls {
     // ${REGISTRY} or something.  This has to be threaded through the
     // Shrinkwrap and Node classes carefully, so for now, just treat
     // the default reg as the magical animal that it has been.
-    const resolvedURL = hgi.parseUrl(resolved)
-
-    if (!resolvedURL) {
+    try {
+      const resolvedURL = new URL(resolved)
+
+      if ((this.options.replaceRegistryHost === resolvedURL.hostname) ||
+           this.options.replaceRegistryHost === 'always') {
+        const registryURL = new URL(this.registry)
+        // Replace the host with the registry host while keeping the path intact
+        resolvedURL.hostname = registryURL.hostname
+        resolvedURL.port = registryURL.port
+        return resolvedURL.toString()
+      }
+      return resolved
+    } catch (e) {
       // if we could not parse the url at all then returning nothing
       // here means it will get removed from the tree in the next step
-      return
+      return undefined
     }
-
-    if ((this.options.replaceRegistryHost === resolvedURL.hostname)
-      || this.options.replaceRegistryHost === 'always') {
-      // this.registry always has a trailing slash
-      return `${this.registry.slice(0, -1)}${resolvedURL.pathname}${resolvedURL.searchParams}`
-    }
-
-    return resolved
   }
 
   // bundles are *sort of* like shrinkwraps, in that the branch is defined
@@ -1151,6 +1172,9 @@ module.exports = cls => class Reifier extends cls {
 
       this.#retiredUnchanged[retireFolder] = []
       return promiseAllRejectLate(diff.unchanged.map(node => {
+        if (node.ideallyInert) {
+          return
+        }
         // no need to roll back links, since we'll just delete them anyway
         if (node.isLink) {
           return mkdir(dirname(node.path), { recursive: true, force: true })
@@ -1230,7 +1254,7 @@ module.exports = cls => class Reifier extends cls {
       // skip links that only live within node_modules as they are most
       // likely managed by packages we installed, we only want to rebuild
       // unchanged links we directly manage
-      const linkedFromRoot = node.parent === tree || node.target.fsTop === tree
+      const linkedFromRoot = (node.parent === tree && !node.ideallyInert) || node.target.fsTop === tree
       if (node.isLink && linkedFromRoot) {
         nodes.push(node)
       }

package/lib/edge.js

@@ -206,22 +206,21 @@ class Edge {
     if (this.overrides?.value && this.overrides.value !== '*' && this.overrides.name === this.#name) {
       if (this.overrides.value.startsWith('$')) {
         const ref = this.overrides.value.slice(1)
-        const pkg = this.#from?.sourceReference
+        let pkg = this.#from?.sourceReference
           ? this.#from?.sourceReference.root.package
           : this.#from?.root?.package
-        if (pkg.devDependencies?.[ref]) {
-          return pkg.devDependencies[ref]
-        }
-        if (pkg.optionalDependencies?.[ref]) {
-          return pkg.optionalDependencies[ref]
-        }
-        if (pkg.dependencies?.[ref]) {
-          return pkg.dependencies[ref]
-        }
-        if (pkg.peerDependencies?.[ref]) {
-          return pkg.peerDependencies[ref]
+
+        let specValue = this.#calculateReferentialOverrideSpec(ref, pkg)
+
+        // If the package isn't found in the root package, fall back to the local package.
+        if (!specValue) {
+          pkg = this.#from?.package
+          specValue = this.#calculateReferentialOverrideSpec(ref, pkg)
         }
 
+        if (specValue) {
+          return specValue
+        }
         throw new Error(`Unable to resolve reference ${this.overrides.value}`)
       }
       return this.overrides.value
@@ -229,6 +228,21 @@ class Edge {
     return this.#spec
   }
 
+  #calculateReferentialOverrideSpec (ref, pkg) {
+    if (pkg.devDependencies?.[ref]) {
+      return pkg.devDependencies[ref]
+    }
+    if (pkg.optionalDependencies?.[ref]) {
+      return pkg.optionalDependencies[ref]
+    }
+    if (pkg.dependencies?.[ref]) {
+      return pkg.dependencies[ref]
+    }
+    if (pkg.peerDependencies?.[ref]) {
+      return pkg.peerDependencies[ref]
+    }
+  }
+
   get accept () {
     return this.#accept
   }

package/lib/node.js

@@ -103,6 +103,7 @@ class Node {
       global = false,
       dummy = false,
       sourceReference = null,
+      ideallyInert = false,
     } = options
     // this object gives querySelectorAll somewhere to stash context about a node
     // while processing a query
@@ -197,6 +198,8 @@ class Node {
       this.extraneous = false
     }
 
+    this.ideallyInert = ideallyInert
+
     this.edgesIn = new Set()
     this.edgesOut = new CaseInsensitiveMap()
 
@@ -1074,7 +1077,7 @@ class Node {
   // return true if it's safe to remove this node, because anything that
   // is depending on it would be fine with the thing that they would resolve
   // to if it was removed, or nothing is depending on it in the first place.
-  canDedupe (preferDedupe = false) {
+  canDedupe (preferDedupe = false, explicitRequest = false) {
     // not allowed to mess with shrinkwraps or bundles
     if (this.inDepBundle || this.inShrinkwrap) {
       return false
@@ -1117,6 +1120,11 @@ class Node {
       return true
     }
 
+    // if the other version was an explicit request, then prefer to take the other version
+    if (explicitRequest) {
+      return true
+    }
+
     return false
   }
 

package/lib/place-dep.js

@@ -423,7 +423,7 @@ class PlaceDep {
   // is another satisfying node further up the tree, and if so, dedupes.
   // Even in installStategy is nested, we do this amount of deduplication.
   pruneDedupable (node, descend = true) {
-    if (node.canDedupe(this.preferDedupe)) {
+    if (node.canDedupe(this.preferDedupe, this.explicitRequest)) {
       // gather up all deps that have no valid edges in from outside
       // the dep set, except for this node we're deduping, so that we
       // also prune deps that would be made extraneous.

package/lib/shrinkwrap.js

@@ -109,6 +109,7 @@ const nodeMetaKeys = [
   'inBundle',
   'hasShrinkwrap',
   'hasInstallScript',
+  'ideallyInert',
 ]
 
 const metaFieldFromPkg = (pkg, key) => {
@@ -135,6 +136,10 @@ const assertNoNewer = async (path, data, lockTime, dir, seen) => {
 
   const parent = isParent ? dir : resolve(dir, 'node_modules')
   const rel = relpath(path, dir)
+  const inert = data.packages[rel]?.ideallyInert
+  if (inert) {
+    return
+  }
   seen.add(rel)
   let entries
   if (dir === path) {
@@ -173,7 +178,7 @@ const assertNoNewer = async (path, data, lockTime, dir, seen) => {
 
   // assert that all the entries in the lockfile were seen
   for (const loc in data.packages) {
-    if (!seen.has(loc)) {
+    if (!seen.has(loc) && !data.packages[loc].ideallyInert) {
       throw new Error(`missing from node_modules: ${loc}`)
     }
   }
@@ -783,6 +788,10 @@ class Shrinkwrap {
       // ok, I did my best!  good luck!
     }
 
+    if (lock.ideallyInert) {
+      meta.ideallyInert = true
+    }
+
     if (lock.bundled) {
       meta.inBundle = true
     }
@@ -953,6 +962,12 @@ class Shrinkwrap {
       this.#buildLegacyLockfile(this.tree, this.data)
     }
 
+    if (!this.hiddenLockfile) {
+      for (const node of Object.values(this.data.packages)) {
+        delete node.ideallyInert
+      }
+    }
+
     // lf version 1 = dependencies only
     // lf version 2 = dependencies and packages
     // lf version 3 = packages only

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@npmcli/arborist",
-  "version": "9.0.1",
+  "version": "9.0.2",
   "description": "Manage node_modules trees",
   "dependencies": {
     "@isaacs/string-locale-compare": "^1.1.0",