@npmcli/arborist 9.0.1 → 9.1.0

package/README.md

@@ -267,53 +267,19 @@ are updated by arborist when necessary whenever the tree is modified in
 such a way that the dependency graph can change, and are relevant when
 pruning nodes from the tree.
 
-```
-| extraneous | peer | dev | optional | devOptional | meaning             | prune?            |
-|------------+------+-----+----------+-------------+---------------------+-------------------|
-|            |      |     |          |             | production dep      | never             |
-|------------+------+-----+----------+-------------+---------------------+-------------------|
-|     X      | N/A  | N/A |   N/A    |     N/A     | nothing depends on  | always            |
-|            |      |     |          |             | this, it is trash   |                   |
-|------------+------+-----+----------+-------------+---------------------+-------------------|
-|            |      |  X  |          |      X      | devDependency, or   | if pruning dev    |
-|            |      |     |          | not in lock | only depended upon  |                   |
-|            |      |     |          |             | by devDependencies  |                   |
-|------------+------+-----+----------+-------------+---------------------+-------------------|
-|            |      |     |    X     |      X      | optionalDependency, | if pruning        |
-|            |      |     |          | not in lock | or only depended on | optional          |
-|            |      |     |          |             | by optionalDeps     |                   |
-|------------+------+-----+----------+-------------+---------------------+-------------------|
-|            |      |  X  |    X     |      X      | Optional dependency | if pruning EITHER |
-|            |      |     |          | not in lock | of dep(s) in the    | dev OR optional   |
-|            |      |     |          |             | dev hierarchy       |                   |
-|------------+------+-----+----------+-------------+---------------------+-------------------|
-|            |      |     |          |      X      | BOTH a non-optional | if pruning BOTH   |
-|            |      |     |          |   in lock   | dep within the dev  | dev AND optional  |
-|            |      |     |          |             | hierarchy, AND a    |                   |
-|            |      |     |          |             | dep within the      |                   |
-|            |      |     |          |             | optional hierarchy  |                   |
-|------------+------+-----+----------+-------------+---------------------+-------------------|
-|            |  X   |     |          |             | peer dependency, or | if pruning peers  |
-|            |      |     |          |             | only depended on by |                   |
-|            |      |     |          |             | peer dependencies   |                   |
-|------------+------+-----+----------+-------------+---------------------+-------------------|
-|            |  X   |  X  |          |      X      | peer dependency of  | if pruning peer   |
-|            |      |     |          | not in lock | dev node hierarchy  | OR dev deps       |
-|------------+------+-----+----------+-------------+---------------------+-------------------|
-|            |  X   |     |    X     |      X      | peer dependency of  | if pruning peer   |
-|            |      |     |          | not in lock | optional nodes, or  | OR optional deps  |
-|            |      |     |          |             | peerOptional dep    |                   |
-|------------+------+-----+----------+-------------+---------------------+-------------------|
-|            |  X   |  X  |    X     |      X      | peer optional deps  | if pruning peer   |
-|            |      |     |          | not in lock | of the dev dep      | OR optional OR    |
-|            |      |     |          |             | hierarchy           | dev               |
-|------------+------+-----+----------+-------------+---------------------+-------------------|
-|            |  X   |     |          |      X      | BOTH a non-optional | if pruning peers  |
-|            |      |     |          |   in lock   | peer dep within the | OR:               |
-|            |      |     |          |             | dev hierarchy, AND  | BOTH optional     |
-|            |      |     |          |             | a peer optional dep | AND dev deps      |
-+------------+------+-----+----------+-------------+---------------------+-------------------+
-```
+| extraneous | peer | dev | optional | devOptional      | meaning                                                                                          | prune?                                                 |
+|:----------:|:----:|:---:|:--------:|:----------------:|:-------------------------------------------------------------------------------------------------|:-------------------------------------------------------|
+|            |      |     |          |                  | production dep                                                                                   | never                                                  |
+|   X        | N/A  | N/A |   N/A    |     N/A          | nothing depends on this, it is trash                                                             | always                                                 |
+|            |      |  X  |          | X<br>not in lock | devDependency, or only depended<br>on by devDependencies                                         | if pruning dev                                         |
+|            |      |     |    X     | X<br>not in lock | optionalDependency, or only depended<br>on by optionalDeps                                       | if pruning optional                                    |
+|            |      |  X  |    X     | X<br>not in lock | Optional dependency of dep(s) in the<br>dev hierarchy                                            | if pruning EITHER<br>dev OR optional                   |
+|            |      |     |          | X<br>in lock     | BOTH a non-optional  dep within the<br>dev hierarchy, AND a dep within<br>the optional hierarchy | if pruning BOTH<br>dev AND optional                    |
+|            |  X   |     |          |                  | peer dependency, or only depended<br>on by peer dependencies                                     | if pruning peers                                       |
+|            |  X   |  X  |          | X<br>not in lock | peer dependency of dev node hierarchy                                                            | if pruning peer OR<br>dev deps                         |
+|            |  X   |     |    X     | X<br>not in lock | peer dependency of optional nodes, or<br>peerOptional dep                                        | if pruning peer OR<br>optional deps                    |
+|            |  X   |  X  |    X     | X<br>not in lock | peer optional deps of the dev dep hierarchy                                                      | if pruning peer OR<br>optional OR dev                  |
+|            |  X   |     |          | X<br>in lock     | BOTH a non-optional peer dep within the<br>dev hierarchy, AND a peer optional dep                | if pruning peer deps OR:<br>BOTH optional AND dev deps |
 
 * If none of these flags are set, then the node is required by the
   dependency and/or peerDependency hierarchy.  It should not be pruned.

package/lib/arborist/build-ideal-tree.js

@@ -13,6 +13,7 @@ const { lstat, readlink } = require('node:fs/promises')
 const { depth } = require('treeverse')
 const { log, time } = require('proc-log')
 const { redact } = require('@npmcli/redact')
+const semver = require('semver')
 
 const {
   OK,
@@ -279,14 +280,23 @@ module.exports = cls => class IdealTreeBuilder extends cls {
       // When updating all, we load the shrinkwrap, but don't bother
       // to build out the full virtual tree from it, since we'll be
       // reconstructing it anyway.
-      .then(root => this.options.global ? root
-      : !this[_usePackageLock] || this[_updateAll]
-        ? Shrinkwrap.reset({
-          path: this.path,
-          lockfileVersion: this.options.lockfileVersion,
-          resolveOptions: this.options,
-        }).then(meta => Object.assign(root, { meta }))
-        : this.loadVirtual({ root }))
+      .then(root => {
+        if (this.options.global) {
+          return root
+        } else if (!this[_usePackageLock] || this[_updateAll]) {
+          return Shrinkwrap.reset({
+            path: this.path,
+            lockfileVersion: this.options.lockfileVersion,
+            resolveOptions: this.options,
+          }).then(meta => Object.assign(root, { meta }))
+        } else {
+          return this.loadVirtual({ root })
+            .then(tree => {
+              this.#applyRootOverridesToWorkspaces(tree)
+              return tree
+            })
+        }
+      })
 
       // if we don't have a lockfile to go from, then start with the
       // actual tree, so we only make the minimum required changes.
@@ -799,7 +809,8 @@ This is a one-time fix-up, please be patient...
     const crackOpen = this.#complete &&
       node !== this.idealTree &&
       node.resolved &&
-      (hasBundle || hasShrinkwrap)
+      (hasBundle || hasShrinkwrap) &&
+      !node.ideallyInert
     if (crackOpen) {
       const Arborist = this.constructor
       const opt = { ...this.options }
@@ -1475,6 +1486,32 @@ This is a one-time fix-up, please be patient...
     timeEnd()
   }
 
+  #applyRootOverridesToWorkspaces (tree) {
+    const rootOverrides = tree.root.package.overrides || {}
+
+    for (const node of tree.root.inventory.values()) {
+      if (!node.isWorkspace) {
+        continue
+      }
+
+      for (const depName of Object.keys(rootOverrides)) {
+        const edge = node.edgesOut.get(depName)
+        const rootNode = tree.root.children.get(depName)
+
+        // safely skip if either edge or rootNode doesn't exist yet
+        if (!edge || !rootNode) {
+          continue
+        }
+
+        const resolvedRootVersion = rootNode.package.version
+        if (!semver.satisfies(resolvedRootVersion, edge.spec)) {
+          edge.detach()
+          node.children.delete(depName)
+        }
+      }
+    }
+  }
+
   #idealTreePrune () {
     for (const node of this.idealTree.inventory.values()) {
       if (node.extraneous) {
@@ -1491,7 +1528,7 @@ This is a one-time fix-up, please be patient...
 
       const set = optionalSet(node)
       for (const node of set) {
-        node.parent = null
+        node.ideallyInert = true
       }
     }
   }
@@ -1512,6 +1549,7 @@ This is a one-time fix-up, please be patient...
           node.parent !== null
           && !node.isProjectRoot
           && !excludeNodes.has(node)
+          && !node.ideallyInert
         ) {
           this[_addNodeToTrashList](node)
         }

package/lib/arborist/isolated-reifier.js

@@ -81,7 +81,7 @@ module.exports = cls => class IsolatedReifier extends cls {
         }
         queue.push(e.to)
       })
-      if (!next.isProjectRoot && !next.isWorkspace) {
+      if (!next.isProjectRoot && !next.isWorkspace && !next.ideallyInert) {
         root.external.push(await this.externalProxyMemo(next))
       }
     }
@@ -147,8 +147,8 @@ module.exports = cls => class IsolatedReifier extends cls {
     const nonOptionalDeps = edges.filter(e => !e.optional).map(e => e.to.target)
 
     result.localDependencies = await Promise.all(nonOptionalDeps.filter(n => n.isWorkspace).map(this.workspaceProxyMemo))
-    result.externalDependencies = await Promise.all(nonOptionalDeps.filter(n => !n.isWorkspace).map(this.externalProxyMemo))
-    result.externalOptionalDependencies = await Promise.all(optionalDeps.map(this.externalProxyMemo))
+    result.externalDependencies = await Promise.all(nonOptionalDeps.filter(n => !n.isWorkspace && !n.ideallyInert).map(this.externalProxyMemo))
+    result.externalOptionalDependencies = await Promise.all(optionalDeps.filter(n => !n.ideallyInert).map(this.externalProxyMemo))
     result.dependencies = [
       ...result.externalDependencies,
       ...result.localDependencies,

package/lib/arborist/load-virtual.js

@@ -200,6 +200,18 @@ module.exports = cls => class VirtualLoader extends cls {
       const targetPath = resolve(this.path, meta.resolved)
       const targetLoc = relpath(this.path, targetPath)
       const target = nodes.get(targetLoc)
+
+      if (!target) {
+        const err = new Error(
+`Missing target in lock file: "${targetLoc}" is referenced by "${location}" but does not exist.
+To fix:
+1. rm package-lock.json
+2. npm install`
+        )
+        err.code = 'EMISSINGTARGET'
+        throw err
+      }
+
       const link = this.#loadLink(location, targetLoc, target, meta)
       nodes.set(location, link)
       nodes.set(targetLoc, link.target)
@@ -267,6 +279,7 @@ module.exports = cls => class VirtualLoader extends cls {
       integrity: sw.integrity,
       resolved: consistentResolve(sw.resolved, this.path, path),
       pkg: sw,
+      ideallyInert: sw.ideallyInert,
       hasShrinkwrap: sw.hasShrinkwrap,
       dev,
       optional,

package/lib/arborist/reify.js

@@ -8,7 +8,6 @@ const semver = require('semver')
 const debug = require('../debug.js')
 const { walkUp } = require('walk-up-path')
 const { log, time } = require('proc-log')
-const hgi = require('hosted-git-info')
 const rpj = require('read-package-json-fast')
 
 const { dirname, resolve, relative, join } = require('node:path')
@@ -150,7 +149,7 @@ module.exports = cls => class Reifier extends cls {
     for (const path of this[_trashList]) {
       const loc = relpath(this.idealTree.realpath, path)
       const node = this.idealTree.inventory.get(loc)
-      if (node && node.root === this.idealTree) {
+      if (node && node.root === this.idealTree && !node.ideallyInert) {
         node.parent = null
       }
     }
@@ -238,6 +237,18 @@ module.exports = cls => class Reifier extends cls {
     this.idealTree.meta.hiddenLockfile = true
     this.idealTree.meta.lockfileVersion = defaultLockfileVersion
 
+    // Preserve inertness for failed stuff.
+    if (this.actualTree) {
+      for (const [loc, actual] of this.actualTree.inventory.entries()) {
+        if (actual.ideallyInert) {
+          const ideal = this.idealTree.inventory.get(loc)
+          if (ideal) {
+            ideal.ideallyInert = true
+          }
+        }
+      }
+    }
+
     this.actualTree = this.idealTree
     this.idealTree = null
 
@@ -600,6 +611,9 @@ module.exports = cls => class Reifier extends cls {
     // retire the same path at the same time.
     const dirsChecked = new Set()
     return promiseAllRejectLate(leaves.map(async node => {
+      if (node.ideallyInert) {
+        return
+      }
       for (const d of walkUp(node.path)) {
         if (d === node.top.path) {
           break
@@ -744,6 +758,10 @@ module.exports = cls => class Reifier extends cls {
   }
 
   async #extractOrLink (node) {
+    if (node.ideallyInert) {
+      return
+    }
+
     const nm = resolve(node.parent.path, 'node_modules')
     await this.#validateNodeModules(nm)
 
@@ -807,7 +825,14 @@ module.exports = cls => class Reifier extends cls {
     // symlink
     const dir = dirname(node.path)
     const target = node.realpath
-    const rel = relative(dir, target)
+
+    let rel
+    if (node.resolved?.startsWith('file:')) {
+      rel = this.#calculateRelativePath(node, dir, target, nm)
+    } else {
+      rel = relative(dir, target)
+    }
+
     await mkdir(dir, { recursive: true })
     return symlink(rel, node.path, 'junction')
   }
@@ -819,11 +844,42 @@ module.exports = cls => class Reifier extends cls {
       const set = optionalSet(node)
       for (node of set) {
         log.verbose('reify', 'failed optional dependency', node.path)
+        node.ideallyInert = true
         this[_addNodeToTrashList](node)
       }
     }) : p).then(() => node)
   }
 
+  #calculateRelativePath (node, dir, target) {
+    // Check if the node is affected by a root override
+    let hasRootOverride = [...node.edgesIn].some(edge => edge.from.isRoot && edge.overrides)
+    // If not set via edges, see if the root package.json explicitly lists an override
+    if (!hasRootOverride && node.root) {
+      const rootPackage = node.root.target
+      hasRootOverride = !!(rootPackage &&
+        rootPackage.package.overrides &&
+        rootPackage.package.overrides[node.name])
+    }
+    if (!hasRootOverride) {
+      return relative(dir, target)
+    }
+    // If an override is detected, attempt to retrieve the override spec from the root package.json
+    const overrideSpec = node.root?.target?.package?.overrides?.[node.name]
+    if (typeof overrideSpec === 'string' && overrideSpec.startsWith('file:')) {
+      const overridePath = overrideSpec.replace(/^file:/, '')
+      const rootDir = node.root.target.path
+      return relative(dir, resolve(rootDir, overridePath))
+    }
+
+    // Fallback: derive the package name from node.resolved in a platform-agnostic way
+    const filePath = node.resolved.replace(/^file:/, '')
+    // A node.package.name could be different than the folder name
+    const pathParts = filePath.split(/[\\/]/)
+    const packageName = pathParts[pathParts.length - 1]
+
+    return join('..', packageName)
+  }
+
   #registryResolved (resolved) {
     // the default registry url is a magic value meaning "the currently
     // configured registry".
@@ -833,21 +889,33 @@ module.exports = cls => class Reifier extends cls {
     // ${REGISTRY} or something.  This has to be threaded through the
     // Shrinkwrap and Node classes carefully, so for now, just treat
     // the default reg as the magical animal that it has been.
-    const resolvedURL = hgi.parseUrl(resolved)
+    try {
+      const resolvedURL = new URL(resolved)
 
-    if (!resolvedURL) {
+      if ((this.options.replaceRegistryHost === resolvedURL.hostname) ||
+           this.options.replaceRegistryHost === 'always') {
+        const registryURL = new URL(this.registry)
+
+        // Replace the host with the registry host while keeping the path intact
+        resolvedURL.hostname = registryURL.hostname
+        resolvedURL.port = registryURL.port
+
+        // Make sure we don't double-include the path if it's already there
+        const registryPath = registryURL.pathname.replace(/\/$/, '')
+
+        if (registryPath && registryPath !== '/' && !resolvedURL.pathname.startsWith(registryPath)) {
+          // Since hostname is changed, we need to ensure the registry path is included
+          resolvedURL.pathname = registryPath + resolvedURL.pathname
+        }
+
+        return resolvedURL.toString()
+      }
+      return resolved
+    } catch (e) {
       // if we could not parse the url at all then returning nothing
       // here means it will get removed from the tree in the next step
-      return
-    }
-
-    if ((this.options.replaceRegistryHost === resolvedURL.hostname)
-      || this.options.replaceRegistryHost === 'always') {
-      // this.registry always has a trailing slash
-      return `${this.registry.slice(0, -1)}${resolvedURL.pathname}${resolvedURL.searchParams}`
+      return undefined
     }
-
-    return resolved
   }
 
   // bundles are *sort of* like shrinkwraps, in that the branch is defined
@@ -1151,6 +1219,9 @@ module.exports = cls => class Reifier extends cls {
 
       this.#retiredUnchanged[retireFolder] = []
       return promiseAllRejectLate(diff.unchanged.map(node => {
+        if (node.ideallyInert) {
+          return
+        }
         // no need to roll back links, since we'll just delete them anyway
         if (node.isLink) {
           return mkdir(dirname(node.path), { recursive: true, force: true })
@@ -1230,7 +1301,7 @@ module.exports = cls => class Reifier extends cls {
       // skip links that only live within node_modules as they are most
       // likely managed by packages we installed, we only want to rebuild
       // unchanged links we directly manage
-      const linkedFromRoot = node.parent === tree || node.target.fsTop === tree
+      const linkedFromRoot = (node.parent === tree && !node.ideallyInert) || node.target.fsTop === tree
       if (node.isLink && linkedFromRoot) {
         nodes.push(node)
       }

package/lib/edge.js

@@ -206,22 +206,21 @@ class Edge {
     if (this.overrides?.value && this.overrides.value !== '*' && this.overrides.name === this.#name) {
       if (this.overrides.value.startsWith('$')) {
         const ref = this.overrides.value.slice(1)
-        const pkg = this.#from?.sourceReference
+        let pkg = this.#from?.sourceReference
           ? this.#from?.sourceReference.root.package
           : this.#from?.root?.package
-        if (pkg.devDependencies?.[ref]) {
-          return pkg.devDependencies[ref]
-        }
-        if (pkg.optionalDependencies?.[ref]) {
-          return pkg.optionalDependencies[ref]
-        }
-        if (pkg.dependencies?.[ref]) {
-          return pkg.dependencies[ref]
-        }
-        if (pkg.peerDependencies?.[ref]) {
-          return pkg.peerDependencies[ref]
+
+        let specValue = this.#calculateReferentialOverrideSpec(ref, pkg)
+
+        // If the package isn't found in the root package, fall back to the local package.
+        if (!specValue) {
+          pkg = this.#from?.package
+          specValue = this.#calculateReferentialOverrideSpec(ref, pkg)
         }
 
+        if (specValue) {
+          return specValue
+        }
         throw new Error(`Unable to resolve reference ${this.overrides.value}`)
       }
       return this.overrides.value
@@ -229,6 +228,21 @@ class Edge {
     return this.#spec
   }
 
+  #calculateReferentialOverrideSpec (ref, pkg) {
+    if (pkg.devDependencies?.[ref]) {
+      return pkg.devDependencies[ref]
+    }
+    if (pkg.optionalDependencies?.[ref]) {
+      return pkg.optionalDependencies[ref]
+    }
+    if (pkg.dependencies?.[ref]) {
+      return pkg.dependencies[ref]
+    }
+    if (pkg.peerDependencies?.[ref]) {
+      return pkg.peerDependencies[ref]
+    }
+  }
+
   get accept () {
     return this.#accept
   }

package/lib/node.js

@@ -103,6 +103,7 @@ class Node {
       global = false,
       dummy = false,
       sourceReference = null,
+      ideallyInert = false,
     } = options
     // this object gives querySelectorAll somewhere to stash context about a node
     // while processing a query
@@ -197,6 +198,8 @@ class Node {
       this.extraneous = false
     }
 
+    this.ideallyInert = ideallyInert
+
     this.edgesIn = new Set()
     this.edgesOut = new CaseInsensitiveMap()
 
@@ -1074,7 +1077,7 @@ class Node {
   // return true if it's safe to remove this node, because anything that
   // is depending on it would be fine with the thing that they would resolve
   // to if it was removed, or nothing is depending on it in the first place.
-  canDedupe (preferDedupe = false) {
+  canDedupe (preferDedupe = false, explicitRequest = false) {
     // not allowed to mess with shrinkwraps or bundles
     if (this.inDepBundle || this.inShrinkwrap) {
       return false
@@ -1117,6 +1120,11 @@ class Node {
       return true
     }
 
+    // if the other version was an explicit request, then prefer to take the other version
+    if (explicitRequest) {
+      return true
+    }
+
     return false
   }
 

package/lib/place-dep.js

@@ -423,7 +423,7 @@ class PlaceDep {
   // is another satisfying node further up the tree, and if so, dedupes.
   // Even in installStategy is nested, we do this amount of deduplication.
   pruneDedupable (node, descend = true) {
-    if (node.canDedupe(this.preferDedupe)) {
+    if (node.canDedupe(this.preferDedupe, this.explicitRequest)) {
       // gather up all deps that have no valid edges in from outside
       // the dep set, except for this node we're deduping, so that we
       // also prune deps that would be made extraneous.

package/lib/shrinkwrap.js

@@ -109,6 +109,7 @@ const nodeMetaKeys = [
   'inBundle',
   'hasShrinkwrap',
   'hasInstallScript',
+  'ideallyInert',
 ]
 
 const metaFieldFromPkg = (pkg, key) => {
@@ -135,6 +136,10 @@ const assertNoNewer = async (path, data, lockTime, dir, seen) => {
 
   const parent = isParent ? dir : resolve(dir, 'node_modules')
   const rel = relpath(path, dir)
+  const inert = data.packages[rel]?.ideallyInert
+  if (inert) {
+    return
+  }
   seen.add(rel)
   let entries
   if (dir === path) {
@@ -173,7 +178,7 @@ const assertNoNewer = async (path, data, lockTime, dir, seen) => {
 
   // assert that all the entries in the lockfile were seen
   for (const loc in data.packages) {
-    if (!seen.has(loc)) {
+    if (!seen.has(loc) && !data.packages[loc].ideallyInert) {
       throw new Error(`missing from node_modules: ${loc}`)
     }
   }
@@ -783,6 +788,10 @@ class Shrinkwrap {
       // ok, I did my best!  good luck!
     }
 
+    if (lock.ideallyInert) {
+      meta.ideallyInert = true
+    }
+
     if (lock.bundled) {
       meta.inBundle = true
     }
@@ -953,6 +962,12 @@ class Shrinkwrap {
       this.#buildLegacyLockfile(this.tree, this.data)
     }
 
+    if (!this.hiddenLockfile) {
+      for (const node of Object.values(this.data.packages)) {
+        delete node.ideallyInert
+      }
+    }
+
     // lf version 1 = dependencies only
     // lf version 2 = dependencies and packages
     // lf version 3 = packages only

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@npmcli/arborist",
-  "version": "9.0.1",
+  "version": "9.1.0",
   "description": "Manage node_modules trees",
   "dependencies": {
     "@isaacs/string-locale-compare": "^1.1.0",