@npmcli/arborist 9.0.0 → 9.0.2

package/lib/arborist/build-ideal-tree.js

@@ -13,6 +13,7 @@ const { lstat, readlink } = require('node:fs/promises')
 const { depth } = require('treeverse')
 const { log, time } = require('proc-log')
 const { redact } = require('@npmcli/redact')
+const semver = require('semver')
 
 const {
   OK,
@@ -279,14 +280,23 @@ module.exports = cls => class IdealTreeBuilder extends cls {
       // When updating all, we load the shrinkwrap, but don't bother
       // to build out the full virtual tree from it, since we'll be
       // reconstructing it anyway.
-      .then(root => this.options.global ? root
-      : !this[_usePackageLock] || this[_updateAll]
-        ? Shrinkwrap.reset({
-          path: this.path,
-          lockfileVersion: this.options.lockfileVersion,
-          resolveOptions: this.options,
-        }).then(meta => Object.assign(root, { meta }))
-        : this.loadVirtual({ root }))
+      .then(root => {
+        if (this.options.global) {
+          return root
+        } else if (!this[_usePackageLock] || this[_updateAll]) {
+          return Shrinkwrap.reset({
+            path: this.path,
+            lockfileVersion: this.options.lockfileVersion,
+            resolveOptions: this.options,
+          }).then(meta => Object.assign(root, { meta }))
+        } else {
+          return this.loadVirtual({ root })
+            .then(tree => {
+              this.#applyRootOverridesToWorkspaces(tree)
+              return tree
+            })
+        }
+      })
 
       // if we don't have a lockfile to go from, then start with the
       // actual tree, so we only make the minimum required changes.
@@ -447,7 +457,7 @@ module.exports = cls => class IdealTreeBuilder extends cls {
             .catch(/* istanbul ignore next */ () => null)
           if (st && st.isSymbolicLink()) {
             const target = await readlink(dir)
-            const real = resolve(dirname(dir), target).replace(/#/g, '%23')
+            const real = resolve(dirname(dir), target)
             tree.package.dependencies[name] = `file:${real}`
           } else {
             tree.package.dependencies[name] = '*'
@@ -522,12 +532,12 @@ module.exports = cls => class IdealTreeBuilder extends cls {
 
       const { name } = spec
       if (spec.type === 'file') {
-        spec = npa(`file:${relpath(path, spec.fetchSpec).replace(/#/g, '%23')}`, path)
+        spec = npa(`file:${relpath(path, spec.fetchSpec)}`, path)
         spec.name = name
       } else if (spec.type === 'directory') {
         try {
           const real = await realpath(spec.fetchSpec, this[_rpcache], this[_stcache])
-          spec = npa(`file:${relpath(path, real).replace(/#/g, '%23')}`, path)
+          spec = npa(`file:${relpath(path, real)}`, path)
           spec.name = name
         } catch {
           // TODO: create synthetic test case to simulate realpath failure
@@ -799,7 +809,8 @@ This is a one-time fix-up, please be patient...
     const crackOpen = this.#complete &&
       node !== this.idealTree &&
       node.resolved &&
-      (hasBundle || hasShrinkwrap)
+      (hasBundle || hasShrinkwrap) &&
+      !node.ideallyInert
     if (crackOpen) {
       const Arborist = this.constructor
       const opt = { ...this.options }
@@ -1475,6 +1486,32 @@ This is a one-time fix-up, please be patient...
     timeEnd()
   }
 
+  #applyRootOverridesToWorkspaces (tree) {
+    const rootOverrides = tree.root.package.overrides || {}
+
+    for (const node of tree.root.inventory.values()) {
+      if (!node.isWorkspace) {
+        continue
+      }
+
+      for (const depName of Object.keys(rootOverrides)) {
+        const edge = node.edgesOut.get(depName)
+        const rootNode = tree.root.children.get(depName)
+
+        // safely skip if either edge or rootNode doesn't exist yet
+        if (!edge || !rootNode) {
+          continue
+        }
+
+        const resolvedRootVersion = rootNode.package.version
+        if (!semver.satisfies(resolvedRootVersion, edge.spec)) {
+          edge.detach()
+          node.children.delete(depName)
+        }
+      }
+    }
+  }
+
   #idealTreePrune () {
     for (const node of this.idealTree.inventory.values()) {
       if (node.extraneous) {
@@ -1491,7 +1528,7 @@ This is a one-time fix-up, please be patient...
 
       const set = optionalSet(node)
       for (const node of set) {
-        node.parent = null
+        node.ideallyInert = true
       }
     }
   }
@@ -1512,6 +1549,7 @@ This is a one-time fix-up, please be patient...
           node.parent !== null
           && !node.isProjectRoot
           && !excludeNodes.has(node)
+          && !node.ideallyInert
         ) {
           this[_addNodeToTrashList](node)
         }

package/lib/arborist/isolated-reifier.js

@@ -81,7 +81,7 @@ module.exports = cls => class IsolatedReifier extends cls {
         }
         queue.push(e.to)
       })
-      if (!next.isProjectRoot && !next.isWorkspace) {
+      if (!next.isProjectRoot && !next.isWorkspace && !next.ideallyInert) {
         root.external.push(await this.externalProxyMemo(next))
       }
     }
@@ -147,8 +147,8 @@ module.exports = cls => class IsolatedReifier extends cls {
     const nonOptionalDeps = edges.filter(e => !e.optional).map(e => e.to.target)
 
     result.localDependencies = await Promise.all(nonOptionalDeps.filter(n => n.isWorkspace).map(this.workspaceProxyMemo))
-    result.externalDependencies = await Promise.all(nonOptionalDeps.filter(n => !n.isWorkspace).map(this.externalProxyMemo))
-    result.externalOptionalDependencies = await Promise.all(optionalDeps.map(this.externalProxyMemo))
+    result.externalDependencies = await Promise.all(nonOptionalDeps.filter(n => !n.isWorkspace && !n.ideallyInert).map(this.externalProxyMemo))
+    result.externalOptionalDependencies = await Promise.all(optionalDeps.filter(n => !n.ideallyInert).map(this.externalProxyMemo))
     result.dependencies = [
       ...result.externalDependencies,
       ...result.localDependencies,

package/lib/arborist/load-actual.js

@@ -216,7 +216,7 @@ module.exports = cls => class ActualLoader extends cls {
       const actualRoot = tree.isLink ? tree.target : tree
       const { dependencies = {} } = actualRoot.package
       for (const [name, kid] of actualRoot.children.entries()) {
-        const def = kid.isLink ? `file:${kid.realpath.replace(/#/g, '%23')}` : '*'
+        const def = kid.isLink ? `file:${kid.realpath}` : '*'
         dependencies[name] = dependencies[name] || def
       }
       actualRoot.package = { ...actualRoot.package, dependencies }

package/lib/arborist/load-virtual.js

@@ -149,7 +149,7 @@ module.exports = cls => class VirtualLoader extends cls {
     })
 
     for (const [name, path] of workspaces.entries()) {
-      lockWS[name] = `file:${path.replace(/#/g, '%23')}`
+      lockWS[name] = `file:${path}`
     }
 
     // Should rootNames exclude optional?
@@ -267,6 +267,7 @@ module.exports = cls => class VirtualLoader extends cls {
       integrity: sw.integrity,
       resolved: consistentResolve(sw.resolved, this.path, path),
       pkg: sw,
+      ideallyInert: sw.ideallyInert,
       hasShrinkwrap: sw.hasShrinkwrap,
       dev,
       optional,

package/lib/arborist/reify.js

@@ -8,7 +8,6 @@ const semver = require('semver')
 const debug = require('../debug.js')
 const { walkUp } = require('walk-up-path')
 const { log, time } = require('proc-log')
-const hgi = require('hosted-git-info')
 const rpj = require('read-package-json-fast')
 
 const { dirname, resolve, relative, join } = require('node:path')
@@ -150,7 +149,7 @@ module.exports = cls => class Reifier extends cls {
     for (const path of this[_trashList]) {
       const loc = relpath(this.idealTree.realpath, path)
       const node = this.idealTree.inventory.get(loc)
-      if (node && node.root === this.idealTree) {
+      if (node && node.root === this.idealTree && !node.ideallyInert) {
         node.parent = null
       }
     }
@@ -238,6 +237,18 @@ module.exports = cls => class Reifier extends cls {
     this.idealTree.meta.hiddenLockfile = true
     this.idealTree.meta.lockfileVersion = defaultLockfileVersion
 
+    // Preserve inertness for failed stuff.
+    if (this.actualTree) {
+      for (const [loc, actual] of this.actualTree.inventory.entries()) {
+        if (actual.ideallyInert) {
+          const ideal = this.idealTree.inventory.get(loc)
+          if (ideal) {
+            ideal.ideallyInert = true
+          }
+        }
+      }
+    }
+
     this.actualTree = this.idealTree
     this.idealTree = null
 
@@ -600,6 +611,9 @@ module.exports = cls => class Reifier extends cls {
     // retire the same path at the same time.
     const dirsChecked = new Set()
     return promiseAllRejectLate(leaves.map(async node => {
+      if (node.ideallyInert) {
+        return
+      }
       for (const d of walkUp(node.path)) {
         if (d === node.top.path) {
           break
@@ -744,6 +758,10 @@ module.exports = cls => class Reifier extends cls {
   }
 
   async #extractOrLink (node) {
+    if (node.ideallyInert) {
+      return
+    }
+
     const nm = resolve(node.parent.path, 'node_modules')
     await this.#validateNodeModules(nm)
 
@@ -819,6 +837,7 @@ module.exports = cls => class Reifier extends cls {
       const set = optionalSet(node)
       for (node of set) {
         log.verbose('reify', 'failed optional dependency', node.path)
+        node.ideallyInert = true
         this[_addNodeToTrashList](node)
       }
     }) : p).then(() => node)
@@ -833,21 +852,23 @@ module.exports = cls => class Reifier extends cls {
     // ${REGISTRY} or something.  This has to be threaded through the
     // Shrinkwrap and Node classes carefully, so for now, just treat
     // the default reg as the magical animal that it has been.
-    const resolvedURL = hgi.parseUrl(resolved)
-
-    if (!resolvedURL) {
+    try {
+      const resolvedURL = new URL(resolved)
+
+      if ((this.options.replaceRegistryHost === resolvedURL.hostname) ||
+           this.options.replaceRegistryHost === 'always') {
+        const registryURL = new URL(this.registry)
+        // Replace the host with the registry host while keeping the path intact
+        resolvedURL.hostname = registryURL.hostname
+        resolvedURL.port = registryURL.port
+        return resolvedURL.toString()
+      }
+      return resolved
+    } catch (e) {
       // if we could not parse the url at all then returning nothing
       // here means it will get removed from the tree in the next step
-      return
+      return undefined
     }
-
-    if ((this.options.replaceRegistryHost === resolvedURL.hostname)
-      || this.options.replaceRegistryHost === 'always') {
-      // this.registry always has a trailing slash
-      return `${this.registry.slice(0, -1)}${resolvedURL.pathname}${resolvedURL.searchParams}`
-    }
-
-    return resolved
   }
 
   // bundles are *sort of* like shrinkwraps, in that the branch is defined
@@ -1151,6 +1172,9 @@ module.exports = cls => class Reifier extends cls {
 
       this.#retiredUnchanged[retireFolder] = []
       return promiseAllRejectLate(diff.unchanged.map(node => {
+        if (node.ideallyInert) {
+          return
+        }
         // no need to roll back links, since we'll just delete them anyway
         if (node.isLink) {
           return mkdir(dirname(node.path), { recursive: true, force: true })
@@ -1230,7 +1254,7 @@ module.exports = cls => class Reifier extends cls {
       // skip links that only live within node_modules as they are most
       // likely managed by packages we installed, we only want to rebuild
       // unchanged links we directly manage
-      const linkedFromRoot = node.parent === tree || node.target.fsTop === tree
+      const linkedFromRoot = (node.parent === tree && !node.ideallyInert) || node.target.fsTop === tree
       if (node.isLink && linkedFromRoot) {
         nodes.push(node)
       }
@@ -1364,7 +1388,7 @@ module.exports = cls => class Reifier extends cls {
             // path initially, in which case we can end up with the wrong
             // thing, so just get the ultimate fetchSpec and relativize it.
             const p = req.fetchSpec.replace(/^file:/, '')
-            const rel = relpath(addTree.realpath, p).replace(/#/g, '%23')
+            const rel = relpath(addTree.realpath, p)
             newSpec = `file:${rel}`
           }
         } else {

package/lib/consistent-resolve.js

@@ -20,11 +20,10 @@ const consistentResolve = (resolved, fromPath, toPath, relPaths = false) => {
       raw,
     } = npa(resolved, fromPath)
     if (type === 'file' || type === 'directory') {
-      const cleanFetchSpec = fetchSpec.replace(/#/g, '%23')
       if (relPaths && toPath) {
-        return `file:${relpath(toPath, cleanFetchSpec)}`
+        return `file:${relpath(toPath, fetchSpec)}`
       }
-      return `file:${cleanFetchSpec}`
+      return `file:${fetchSpec}`
     }
     if (hosted) {
       return `git+${hosted.auth ? hosted.https(hostedOpt) : hosted.sshurl(hostedOpt)}`

package/lib/dep-valid.js

@@ -101,7 +101,7 @@ const depValid = (child, requested, requestor) => {
       })
     }
 
-    default: // unpossible, just being cautious
+    default: // impossible, just being cautious
       break
   }
 

package/lib/edge.js

@@ -4,6 +4,7 @@
 const util = require('node:util')
 const npa = require('npm-package-arg')
 const depValid = require('./dep-valid.js')
+const OverrideSet = require('./override-set.js')
 
 class ArboristEdge {
   constructor (edge) {
@@ -103,7 +104,7 @@ class Edge {
   }
 
   satisfiedBy (node) {
-    if (node.name !== this.#name) {
+    if (node.name !== this.#name || !this.#from) {
       return false
     }
 
@@ -112,7 +113,31 @@ class Edge {
     if (node.hasShrinkwrap || node.inShrinkwrap || node.inBundle) {
       return depValid(node, this.rawSpec, this.#accept, this.#from)
     }
-    return depValid(node, this.spec, this.#accept, this.#from)
+
+    // If there's no override we just use the spec.
+    if (!this.overrides?.keySpec) {
+      return depValid(node, this.spec, this.#accept, this.#from)
+    }
+    // There's some override. If the target node satisfies the overriding spec
+    // then it's okay.
+    if (depValid(node, this.spec, this.#accept, this.#from)) {
+      return true
+    }
+    // If it doesn't, then it should at least satisfy the original spec.
+    if (!depValid(node, this.rawSpec, this.#accept, this.#from)) {
+      return false
+    }
+    // It satisfies the original spec, not the overriding spec. We need to make
+    // sure it doesn't use the overridden spec.
+    // For example:
+    //   we might have an ^8.0.0 rawSpec, and an override that makes
+    //   keySpec=8.23.0 and the override value spec=9.0.0.
+    //   If the node is 9.0.0, then it's okay because it's consistent with spec.
+    //   If the node is 8.24.0, then it's okay because it's consistent with the rawSpec.
+    //   If the node is 8.23.0, then it's not okay because even though it's consistent
+    //   with the rawSpec, it's also consistent with the keySpec.
+    //   So we're looking for ^8.0.0 or 9.0.0 and not 8.23.0.
+    return !depValid(node, this.overrides.keySpec, this.#accept, this.#from)
   }
 
   // return the edge data, and an explanation of how that edge came to be here
@@ -181,24 +206,21 @@ class Edge {
     if (this.overrides?.value && this.overrides.value !== '*' && this.overrides.name === this.#name) {
       if (this.overrides.value.startsWith('$')) {
         const ref = this.overrides.value.slice(1)
-        // we may be a virtual root, if we are we want to resolve reference overrides
-        // from the real root, not the virtual one
-        const pkg = this.#from.sourceReference
-          ? this.#from.sourceReference.root.package
-          : this.#from.root.package
-        if (pkg.devDependencies?.[ref]) {
-          return pkg.devDependencies[ref]
-        }
-        if (pkg.optionalDependencies?.[ref]) {
-          return pkg.optionalDependencies[ref]
-        }
-        if (pkg.dependencies?.[ref]) {
-          return pkg.dependencies[ref]
-        }
-        if (pkg.peerDependencies?.[ref]) {
-          return pkg.peerDependencies[ref]
+        let pkg = this.#from?.sourceReference
+          ? this.#from?.sourceReference.root.package
+          : this.#from?.root?.package
+
+        let specValue = this.#calculateReferentialOverrideSpec(ref, pkg)
+
+        // If the package isn't found in the root package, fall back to the local package.
+        if (!specValue) {
+          pkg = this.#from?.package
+          specValue = this.#calculateReferentialOverrideSpec(ref, pkg)
         }
 
+        if (specValue) {
+          return specValue
+        }
         throw new Error(`Unable to resolve reference ${this.overrides.value}`)
       }
       return this.overrides.value
@@ -206,6 +228,21 @@ class Edge {
     return this.#spec
   }
 
+  #calculateReferentialOverrideSpec (ref, pkg) {
+    if (pkg.devDependencies?.[ref]) {
+      return pkg.devDependencies[ref]
+    }
+    if (pkg.optionalDependencies?.[ref]) {
+      return pkg.optionalDependencies[ref]
+    }
+    if (pkg.dependencies?.[ref]) {
+      return pkg.dependencies[ref]
+    }
+    if (pkg.peerDependencies?.[ref]) {
+      return pkg.peerDependencies[ref]
+    }
+  }
+
   get accept () {
     return this.#accept
   }
@@ -234,10 +271,15 @@ class Edge {
         } else {
           this.#error = 'MISSING'
         }
-      } else if (this.peer && this.#from === this.#to.parent && !this.#from.isTop) {
+      } else if (this.peer && this.#from === this.#to.parent && !this.#from?.isTop) {
         this.#error = 'PEER LOCAL'
       } else if (!this.satisfiedBy(this.#to)) {
         this.#error = 'INVALID'
+      } else if (this.overrides && this.#to.edgesOut.size && OverrideSet.doOverrideSetsConflict(this.overrides, this.#to.overrides)) {
+        // Any inconsistency between the edge's override set and the target's override set is potentially problematic.
+        // But we only say the edge is in error if the override sets are plainly conflicting.
+        // Note that if the target doesn't have any dependencies of their own, then this inconsistency is irrelevant.
+        this.#error = 'INVALID'
       } else {
         this.#error = 'OK'
       }
@@ -250,15 +292,26 @@ class Edge {
 
   reload (hard = false) {
     this.#explanation = null
-    if (this.#from.overrides) {
-      this.overrides = this.#from.overrides.getEdgeRule(this)
+
+    let needToUpdateOverrideSet = false
+    let newOverrideSet
+    let oldOverrideSet
+    if (this.#from?.overrides) {
+      newOverrideSet = this.#from.overrides.getEdgeRule(this)
+      if (newOverrideSet && !newOverrideSet.isEqual(this.overrides)) {
+        // If there's a new different override set we need to propagate it to the nodes.
+        // If we're deleting the override set then there's no point propagating it right now since it will be filled with another value later.
+        needToUpdateOverrideSet = true
+        oldOverrideSet = this.overrides
+        this.overrides = newOverrideSet
+      }
     } else {
       delete this.overrides
     }
-    const newTo = this.#from.resolve(this.#name)
+    const newTo = this.#from?.resolve(this.#name)
     if (newTo !== this.#to) {
       if (this.#to) {
-        this.#to.edgesIn.delete(this)
+        this.#to.deleteEdgeIn(this)
       }
       this.#to = newTo
       this.#error = null
@@ -267,15 +320,19 @@ class Edge {
       }
     } else if (hard) {
       this.#error = null
+    } else if (needToUpdateOverrideSet && this.#to) {
+      // Propagate the new override set to the target node.
+      this.#to.updateOverridesEdgeInRemoved(oldOverrideSet)
+      this.#to.updateOverridesEdgeInAdded(newOverrideSet)
     }
   }
 
   detach () {
     this.#explanation = null
     if (this.#to) {
-      this.#to.edgesIn.delete(this)
+      this.#to.deleteEdgeIn(this)
     }
-    this.#from.edgesOut.delete(this.#name)
+    this.#from?.edgesOut.delete(this.#name)
     this.#to = null
     this.#error = 'DETACHED'
     this.#from = null

package/lib/link.js

@@ -99,7 +99,7 @@ class Link extends Node {
     // the path/realpath guard is there for the benefit of setting
     // these things in the "wrong" order
     return this.path && this.realpath
-      ? `file:${relpath(dirname(this.path), this.realpath).replace(/#/g, '%23')}`
+      ? `file:${relpath(dirname(this.path), this.realpath)}`
       : null
   }
 

package/lib/node.js

@@ -40,6 +40,7 @@ const debug = require('./debug.js')
 const gatherDepSet = require('./gather-dep-set.js')
 const treeCheck = require('./tree-check.js')
 const { walkUp } = require('walk-up-path')
+const { log } = require('proc-log')
 
 const { resolve, relative, dirname, basename } = require('node:path')
 const util = require('node:util')
@@ -102,6 +103,7 @@ class Node {
       global = false,
       dummy = false,
       sourceReference = null,
+      ideallyInert = false,
     } = options
     // this object gives querySelectorAll somewhere to stash context about a node
     // while processing a query
@@ -196,6 +198,8 @@ class Node {
       this.extraneous = false
     }
 
+    this.ideallyInert = ideallyInert
+
     this.edgesIn = new Set()
     this.edgesOut = new CaseInsensitiveMap()
 
@@ -344,7 +348,28 @@ class Node {
   }
 
   get overridden () {
-    return !!(this.overrides && this.overrides.value && this.overrides.name === this.name)
+    if (!this.overrides) {
+      return false
+    }
+    if (!this.overrides.value) {
+      return false
+    }
+    if (this.overrides.name !== this.name) {
+      return false
+    }
+
+    // The overrides rule is for a package with this name, but some override rules only apply to specific
+    // versions. To make sure this package was actually overridden, we check whether any edge going in
+    // had the rule applied to it, in which case its overrides set is different than its source node.
+    for (const edge of this.edgesIn) {
+      if (edge.overrides && edge.overrides.name === this.name && edge.overrides.value === this.version) {
+        if (!edge.overrides.isEqual(edge.from.overrides)) {
+          return true
+        }
+      }
+    }
+
+    return false
   }
 
   get package () {
@@ -822,9 +847,6 @@ class Node {
       target.root = root
     }
 
-    if (!this.overrides && this.parent && this.parent.overrides) {
-      this.overrides = this.parent.overrides.getNodeRule(this)
-    }
     // tree should always be valid upon root setter completion.
     treeCheck(this)
     if (this !== root) {
@@ -842,7 +864,7 @@ class Node {
     }
 
     for (const [name, path] of this.#workspaces.entries()) {
-      new Edge({ from: this, name, spec: `file:${path.replace(/#/g, '%23')}`, type: 'workspace' })
+      new Edge({ from: this, name, spec: `file:${path}`, type: 'workspace' })
     }
   }
 
@@ -1006,10 +1028,21 @@ class Node {
       return false
     }
 
-    // XXX need to check for two root nodes?
-    if (node.overrides !== this.overrides) {
-      return false
+    // If this node has no dependencies, then it's irrelevant to check the override
+    // rules of the replacement node.
+    if (this.edgesOut.size) {
+      // XXX need to check for two root nodes?
+      if (node.overrides) {
+        if (!node.overrides.isEqual(this.overrides)) {
+          return false
+        }
+      } else {
+        if (this.overrides) {
+          return false
+        }
+      }
     }
+
     ignorePeers = new Set(ignorePeers)
 
     // gather up all the deps of this node and that are only depended
@@ -1044,7 +1077,7 @@ class Node {
   // return true if it's safe to remove this node, because anything that
   // is depending on it would be fine with the thing that they would resolve
   // to if it was removed, or nothing is depending on it in the first place.
-  canDedupe (preferDedupe = false) {
+  canDedupe (preferDedupe = false, explicitRequest = false) {
     // not allowed to mess with shrinkwraps or bundles
     if (this.inDepBundle || this.inShrinkwrap) {
       return false
@@ -1077,8 +1110,18 @@ class Node {
       return false
     }
 
-    // if we prefer dedupe, or if the version is greater/equal, take the other
-    if (preferDedupe || semver.gte(other.version, this.version)) {
+    // if we prefer dedupe, or if the version is equal, take the other
+    if (preferDedupe || semver.eq(other.version, this.version)) {
+      return true
+    }
+
+    // if our current version isn't the result of an override, then prefer to take the greater version
+    if (!this.overridden && semver.gt(other.version, this.version)) {
+      return true
+    }
+
+    // if the other version was an explicit request, then prefer to take the other version
+    if (explicitRequest) {
       return true
     }
 
@@ -1249,10 +1292,6 @@ class Node {
       this[_changePath](newPath)
     }
 
-    if (parent.overrides) {
-      this.overrides = parent.overrides.getNodeRule(this)
-    }
-
     // clobbers anything at that path, resets all appropriate references
     this.root = parent.root
   }
@@ -1346,9 +1385,87 @@ class Node {
     this.edgesOut.set(edge.name, edge)
   }
 
-  addEdgeIn (edge) {
+  recalculateOutEdgesOverrides () {
+    // For each edge out propogate the new overrides through.
+    for (const edge of this.edgesOut.values()) {
+      edge.reload(true)
+      if (edge.to) {
+        edge.to.updateOverridesEdgeInAdded(edge.overrides)
+      }
+    }
+  }
+
+  updateOverridesEdgeInRemoved (otherOverrideSet) {
+    // If this edge's overrides isn't equal to this node's overrides, then removing it won't change newOverrideSet later.
+    if (!this.overrides || !this.overrides.isEqual(otherOverrideSet)) {
+      return false
+    }
+    let newOverrideSet
+    for (const edge of this.edgesIn) {
+      if (newOverrideSet && edge.overrides) {
+        newOverrideSet = OverrideSet.findSpecificOverrideSet(edge.overrides, newOverrideSet)
+      } else {
+        newOverrideSet = edge.overrides
+      }
+    }
+    if (this.overrides.isEqual(newOverrideSet)) {
+      return false
+    }
+    this.overrides = newOverrideSet
+    if (this.overrides) {
+      // Optimization: if there's any override set at all, then no non-extraneous node has an empty override set. So if we temporarily have no
+      // override set (for example, we removed all the edges in), there's no use updating all the edges out right now. Let's just wait until
+      // we have an actual override set later.
+      this.recalculateOutEdgesOverrides()
+    }
+    return true
+  }
+
+  // This logic isn't perfect either. When we have two edges in that have different override sets, then we have to decide which set is correct.
+  // This function assumes the more specific override set is applicable, so if we have dependencies A->B->C and A->C
+  // and an override set that specifies what happens for C under A->B, this will work even if the new A->C edge comes along and tries to change
+  // the override set.
+  // The strictly correct logic is not to allow two edges with different overrides to point to the same node, because even if this node can satisfy
+  // both, one of its dependencies might need to be different depending on the edge leading to it.
+  // However, this might cause a lot of duplication, because the conflict in the dependencies might never actually happen.
+  updateOverridesEdgeInAdded (otherOverrideSet) {
+    if (!otherOverrideSet) {
+      // Assuming there are any overrides at all, the overrides field is never undefined for any node at the end state of the tree.
+      // So if the new edge's overrides is undefined it will be updated later. So we can wait with updating the node's overrides field.
+      return false
+    }
+    if (!this.overrides) {
+      this.overrides = otherOverrideSet
+      this.recalculateOutEdgesOverrides()
+      return true
+    }
+    if (this.overrides.isEqual(otherOverrideSet)) {
+      return false
+    }
+    const newOverrideSet = OverrideSet.findSpecificOverrideSet(this.overrides, otherOverrideSet)
+    if (newOverrideSet) {
+      if (!this.overrides.isEqual(newOverrideSet)) {
+        this.overrides = newOverrideSet
+        this.recalculateOutEdgesOverrides()
+        return true
+      }
+      return false
+    }
+    // This is an error condition. We can only get here if the new override set is in conflict with the existing.
+    log.silly('Conflicting override sets', this.name)
+  }
+
+  deleteEdgeIn (edge) {
+    this.edgesIn.delete(edge)
     if (edge.overrides) {
-      this.overrides = edge.overrides
+      this.updateOverridesEdgeInRemoved(edge.overrides)
+    }
+  }
+
+  addEdgeIn (edge) {
+    // We need to handle the case where the new edge in has an overrides field which is different from the current value.
+    if (!this.overrides || !this.overrides.isEqual(edge.overrides)) {
+      this.updateOverridesEdgeInAdded(edge.overrides)
     }
 
     this.edgesIn.add(edge)

package/lib/override-set.js

@@ -1,5 +1,6 @@
 const npa = require('npm-package-arg')
 const semver = require('semver')
+const { log } = require('proc-log')
 
 class OverrideSet {
   constructor ({ overrides, key, parent }) {
@@ -44,6 +45,43 @@ class OverrideSet {
     }
   }
 
+  childrenAreEqual (other) {
+    if (this.children.size !== other.children.size) {
+      return false
+    }
+    for (const [key] of this.children) {
+      if (!other.children.has(key)) {
+        return false
+      }
+      if (this.children.get(key).value !== other.children.get(key).value) {
+        return false
+      }
+      if (!this.children.get(key).childrenAreEqual(other.children.get(key))) {
+        return false
+      }
+    }
+    return true
+  }
+
+  isEqual (other) {
+    if (this === other) {
+      return true
+    }
+    if (!other) {
+      return false
+    }
+    if (this.key !== other.key || this.value !== other.value) {
+      return false
+    }
+    if (!this.childrenAreEqual(other)) {
+      return false
+    }
+    if (!this.parent) {
+      return !other.parent
+    }
+    return this.parent.isEqual(other.parent)
+  }
+
   getEdgeRule (edge) {
     for (const rule of this.ruleset.values()) {
       if (rule.name !== edge.name) {
@@ -55,7 +93,9 @@ class OverrideSet {
         return rule
       }
 
-      let spec = npa(`${edge.name}@${edge.spec}`)
+      // We need to use the rawSpec here, because the spec has the overrides applied to it already.
+      // rawSpec can be undefined, so we need to use the fallback value of spec if it is.
+      let spec = npa(`${edge.name}@${edge.rawSpec || edge.spec}`)
       if (spec.type === 'alias') {
         spec = spec.subSpec
       }
@@ -142,6 +182,28 @@ class OverrideSet {
 
     return ruleset
   }
+
+  static findSpecificOverrideSet (first, second) {
+    for (let overrideSet = second; overrideSet; overrideSet = overrideSet.parent) {
+      if (overrideSet.isEqual(first)) {
+        return second
+      }
+    }
+    for (let overrideSet = first; overrideSet; overrideSet = overrideSet.parent) {
+      if (overrideSet.isEqual(second)) {
+        return first
+      }
+    }
+
+    // The override sets are incomparable. Neither one contains the other.
+    log.silly('Conflicting override sets', first, second)
+  }
+
+  static doOverrideSetsConflict (first, second) {
+    // If override sets contain one another then we can try to use the more specific one.
+    // If neither one is more specific, then we consider them to be in conflict.
+    return (this.findSpecificOverrideSet(first, second) === undefined)
+  }
 }
 
 module.exports = OverrideSet

package/lib/place-dep.js

@@ -423,7 +423,7 @@ class PlaceDep {
   // is another satisfying node further up the tree, and if so, dedupes.
   // Even in installStategy is nested, we do this amount of deduplication.
   pruneDedupable (node, descend = true) {
-    if (node.canDedupe(this.preferDedupe)) {
+    if (node.canDedupe(this.preferDedupe, this.explicitRequest)) {
       // gather up all deps that have no valid edges in from outside
       // the dep set, except for this node we're deduping, so that we
       // also prune deps that would be made extraneous.

package/lib/shrinkwrap.js

@@ -109,6 +109,7 @@ const nodeMetaKeys = [
   'inBundle',
   'hasShrinkwrap',
   'hasInstallScript',
+  'ideallyInert',
 ]
 
 const metaFieldFromPkg = (pkg, key) => {
@@ -135,6 +136,10 @@ const assertNoNewer = async (path, data, lockTime, dir, seen) => {
 
   const parent = isParent ? dir : resolve(dir, 'node_modules')
   const rel = relpath(path, dir)
+  const inert = data.packages[rel]?.ideallyInert
+  if (inert) {
+    return
+  }
   seen.add(rel)
   let entries
   if (dir === path) {
@@ -173,7 +178,7 @@ const assertNoNewer = async (path, data, lockTime, dir, seen) => {
 
   // assert that all the entries in the lockfile were seen
   for (const loc in data.packages) {
-    if (!seen.has(loc)) {
+    if (!seen.has(loc) && !data.packages[loc].ideallyInert) {
       throw new Error(`missing from node_modules: ${loc}`)
     }
   }
@@ -783,6 +788,10 @@ class Shrinkwrap {
       // ok, I did my best!  good luck!
     }
 
+    if (lock.ideallyInert) {
+      meta.ideallyInert = true
+    }
+
     if (lock.bundled) {
       meta.inBundle = true
     }
@@ -817,7 +826,7 @@ class Shrinkwrap {
         if (!/^file:/.test(resolved)) {
           pathFixed = resolved
         } else {
-          pathFixed = `file:${resolve(this.path, resolved.slice(5)).replace(/#/g, '%23')}`
+          pathFixed = `file:${resolve(this.path, resolved.slice(5))}`
         }
       }
 
@@ -953,6 +962,12 @@ class Shrinkwrap {
       this.#buildLegacyLockfile(this.tree, this.data)
     }
 
+    if (!this.hiddenLockfile) {
+      for (const node of Object.values(this.data.packages)) {
+        delete node.ideallyInert
+      }
+    }
+
     // lf version 1 = dependencies only
     // lf version 2 = dependencies and packages
     // lf version 3 = packages only
@@ -1011,7 +1026,7 @@ class Shrinkwrap {
     }
 
     if (node.isLink) {
-      lock.version = `file:${relpath(this.path, node.realpath).replace(/#/g, '%23')}`
+      lock.version = `file:${relpath(this.path, node.realpath)}`
     } else if (spec && (spec.type === 'file' || spec.type === 'remote')) {
       lock.version = spec.saveSpec
     } else if (spec && spec.type === 'git' || rSpec.type === 'git') {
@@ -1089,7 +1104,7 @@ class Shrinkwrap {
             // this especially shows up with workspace edges when the root
             // node is also a workspace in the set.
             const p = resolve(node.realpath, spec.slice('file:'.length))
-            set[k] = `file:${relpath(node.realpath, p).replace(/#/g, '%23')}`
+            set[k] = `file:${relpath(node.realpath, p)}`
           } else {
             set[k] = spec
           }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@npmcli/arborist",
-  "version": "9.0.0",
+  "version": "9.0.2",
   "description": "Manage node_modules trees",
   "dependencies": {
     "@isaacs/string-locale-compare": "^1.1.0",