@npmcli/arborist 9.1.5 → 9.1.7

package/lib/arborist/reify.js

@@ -7,7 +7,6 @@ const pacote = require('pacote')
 const promiseAllRejectLate = require('promise-all-reject-late')
 const runScript = require('@npmcli/run-script')
 const { callLimit: promiseCallLimit } = require('promise-call-limit')
-const { checkEngine, checkPlatform } = require('npm-install-checks')
 const { depth: dfwalk } = require('treeverse')
 const { dirname, resolve, relative, join } = require('node:path')
 const { log, time } = require('proc-log')
@@ -58,11 +57,9 @@ const _rollbackRetireShallowNodes = Symbol.for('rollbackRetireShallowNodes')
 const _rollbackCreateSparseTree = Symbol.for('rollbackCreateSparseTree')
 const _rollbackMoveBackRetiredUnchanged = Symbol.for('rollbackMoveBackRetiredUnchanged')
 const _saveIdealTree = Symbol.for('saveIdealTree')
-const _reifyPackages = Symbol.for('reifyPackages')
 
 // defined by build-ideal-tree mixin
 const _resolvedAdd = Symbol.for('resolvedAdd')
-const _usePackageLock = Symbol.for('usePackageLock')
 // used by build-ideal-tree mixin
 const _addNodeToTrashList = Symbol.for('addNodeToTrashList')
 
@@ -71,13 +68,10 @@ const _createIsolatedTree = Symbol.for('createIsolatedTree')
 module.exports = cls => class Reifier extends cls {
   #bundleMissing = new Set() // child nodes we'd EXPECT to be included in a bundle, but aren't
   #bundleUnpacked = new Set() // the nodes we unpack to read their bundles
-  #dryRun
   #nmValidated = new Set()
   #omit
-  #omitted
   #retiredPaths = {}
   #retiredUnchanged = {}
-  #savePrefix
   #shrinkwrapInflated = new Set()
   #sparseTreeDirs = new Set()
   #sparseTreeRoots = new Set()
@@ -99,7 +93,6 @@ module.exports = cls => class Reifier extends cls {
     }
 
     this.#omit = new Set(options.omit)
-    this.#omitted = new Set()
 
     // start tracker block
     this.addTracker('reify')
@@ -125,22 +118,24 @@ module.exports = cls => class Reifier extends cls {
       this.idealTree = await this[_createIsolatedTree]()
     }
     await this[_diffTrees]()
-    await this[_reifyPackages]()
+    await this.#reifyPackages()
     if (linked) {
       // swap back in the idealTree
       // so that the lockfile is preserved
       this.idealTree = oldTree
     }
     await this[_saveIdealTree](options)
-    // clean omitted
-    for (const node of this.#omitted) {
-      node.parent = null
+    // clean inert
+    for (const node of this.idealTree.inventory.values()) {
+      if (node.inert) {
+        node.parent = null
+      }
     }
     // clean up any trash that is still in the tree
     for (const path of this[_trashList]) {
       const loc = relpath(this.idealTree.realpath, path)
       const node = this.idealTree.inventory.get(loc)
-      if (node && node.root === this.idealTree && !node.ideallyInert) {
+      if (node && node.root === this.idealTree) {
         node.parent = null
       }
     }
@@ -228,18 +223,6 @@ module.exports = cls => class Reifier extends cls {
     this.idealTree.meta.hiddenLockfile = true
     this.idealTree.meta.lockfileVersion = defaultLockfileVersion
 
-    // Preserve inertness for failed stuff.
-    if (this.actualTree) {
-      for (const [loc, actual] of this.actualTree.inventory.entries()) {
-        if (actual.ideallyInert) {
-          const ideal = this.idealTree.inventory.get(loc)
-          if (ideal) {
-            ideal.ideallyInert = true
-          }
-        }
-      }
-    }
-
     this.actualTree = this.idealTree
     this.idealTree = null
 
@@ -274,7 +257,7 @@ module.exports = cls => class Reifier extends cls {
     return treeCheck(this.actualTree)
   }
 
-  async [_reifyPackages] () {
+  async #reifyPackages () {
     // we don't submit the audit report or write to disk on dry runs
     if (this.options.dryRun) {
       return
@@ -465,7 +448,6 @@ module.exports = cls => class Reifier extends cls {
     // and ideal trees.
     this.diff = Diff.calculate({
       omit: this.#omit,
-      omitted: this.#omitted,
       shrinkwrapInflated: this.#shrinkwrapInflated,
       filterNodes,
       actual: this.actualTree,
@@ -566,9 +548,6 @@ module.exports = cls => class Reifier extends cls {
     // retire the same path at the same time.
     const dirsChecked = new Set()
     return promiseAllRejectLate(leaves.map(async node => {
-      if (node.ideallyInert) {
-        return
-      }
       for (const d of walkUp(node.path)) {
         if (d === node.top.path) {
           break
@@ -662,18 +641,7 @@ module.exports = cls => class Reifier extends cls {
     const timeEnd = time.start(`reifyNode:${node.location}`)
     this.addTracker('reify', node.name, node.location)
 
-    const { npmVersion, nodeVersion, cpu, os, libc } = this.options
     const p = Promise.resolve().then(async () => {
-      // when we reify an optional node, check the engine and platform
-      // first. be sure to ignore the --force and --engine-strict flags,
-      // since we always want to skip any optional packages we can't install.
-      // these checks throwing will result in a rollback and removal
-      // of the mismatches
-      // eslint-disable-next-line promise/always-return
-      if (node.optional) {
-        checkEngine(node.package, npmVersion, nodeVersion, false)
-        checkPlatform(node.package, false, { cpu, os, libc })
-      }
       await this[_checkBins](node)
       await this.#extractOrLink(node)
       const { _id, deprecated } = node.package
@@ -707,10 +675,6 @@ module.exports = cls => class Reifier extends cls {
   }
 
   async #extractOrLink (node) {
-    if (node.ideallyInert) {
-      return
-    }
-
     const nm = resolve(node.parent.path, 'node_modules')
     await this.#validateNodeModules(nm)
 
@@ -791,9 +755,9 @@ module.exports = cls => class Reifier extends cls {
   [_handleOptionalFailure] (node, p) {
     return (node.optional ? p.catch(() => {
       const set = optionalSet(node)
-      for (node of set) {
+      for (const node of set) {
         log.verbose('reify', 'failed optional dependency', node.path)
-        node.ideallyInert = true
+        node.inert = true
         this[_addNodeToTrashList](node)
       }
     }) : p).then(() => node)
@@ -849,9 +813,17 @@ module.exports = cls => class Reifier extends cls {
         // Make sure we don't double-include the path if it's already there
         const registryPath = registryURL.pathname.replace(/\/$/, '')
 
-        if (registryPath && registryPath !== '/' && !resolvedURL.pathname.startsWith(registryPath)) {
-          // Since hostname is changed, we need to ensure the registry path is included
-          resolvedURL.pathname = registryPath + resolvedURL.pathname
+        if (registryPath && registryPath !== '/') {
+          // Check if the resolved pathname already starts with the registry path
+          // We need to ensure it's a proper path prefix, not just a string prefix
+          // e.g., registry path '/npm' should not match '/npm-run-path'
+          const hasRegistryPath = resolvedURL.pathname === registryPath ||
+                                  resolvedURL.pathname.startsWith(registryPath + '/')
+
+          if (!hasRegistryPath) {
+            // Since hostname is changed, we need to ensure the registry path is included
+            resolvedURL.pathname = registryPath + resolvedURL.pathname
+          }
         }
 
         return resolvedURL.toString()
@@ -1165,9 +1137,6 @@ module.exports = cls => class Reifier extends cls {
 
       this.#retiredUnchanged[retireFolder] = []
       return promiseAllRejectLate(diff.unchanged.map(node => {
-        if (node.ideallyInert) {
-          return
-        }
         // no need to roll back links, since we'll just delete them anyway
         if (node.isLink) {
           return mkdir(dirname(node.path), { recursive: true, force: true })
@@ -1247,7 +1216,7 @@ module.exports = cls => class Reifier extends cls {
       // skip links that only live within node_modules as they are most
       // likely managed by packages we installed, we only want to rebuild
       // unchanged links we directly manage
-      const linkedFromRoot = (node.parent === tree && !node.ideallyInert) || node.target.fsTop === tree
+      const linkedFromRoot = (node.parent === tree && !node.inert) || node.target.fsTop === tree
       if (node.isLink && linkedFromRoot) {
         nodes.push(node)
       }
@@ -1358,7 +1327,7 @@ module.exports = cls => class Reifier extends cls {
           const alias = name !== pname
           newSpec = alias ? `npm:${pname}@${range}` : range
         } else if (req.hosted) {
-          // save the git+https url if it has auth, otherwise shortcut
+          // save the git+https url if it has auth; otherwise, shortcut
           const h = req.hosted
           const opt = { noCommittish: false }
           if (h.https && h.auth) {
@@ -1427,7 +1396,7 @@ module.exports = cls => class Reifier extends cls {
 
     // Returns true if any of the edges from this node has a semver
     // range definition that is an exact match to the version installed
-    // e.g: should return true if for a given an installed version 1.0.0,
+    // e.g: should return true if for a given and installed version 1.0.0,
     // range is either =1.0.0 or 1.0.0
     const exactVersion = node => {
       for (const edge of node.edgesIn) {
@@ -1531,7 +1500,7 @@ module.exports = cls => class Reifier extends cls {
 
     // before now edge specs could be changing, affecting the `requires` field
     // in the package lock, so we hold off saving to the very last action
-    if (this[_usePackageLock]) {
+    if (this.options.usePackageLock) {
       // preserve indentation, if possible
       let format = this.idealTree.package[Symbol.for('indent')]
       if (format === undefined) {

package/lib/calc-dep-flags.js

@@ -1,144 +1,102 @@
-const { depth } = require('treeverse')
-
+// Dep flag (dev, peer, etc.) calculation requires default or reset flags.
+// Flags are true by default and are unset to false as we walk deps.
+// We iterate outward edges looking for dep flags that can
+// be unset based on the current nodes flags and edge type.
+// Examples:
+// - a non-optional node with a non-optional edge out, the edge node should not be optional
+// - a non-peer node with a non-peer edge out, the edge node should not be peer
+// If a node is changed, we add to the queue and continue until no more changes.
+// Flags that remain after all this unsetting should be valid.
+// Examples:
+// - a node still flagged optional must only be reachable via optional edges
+// - a node still flagged peer must only be reachable via peer edges
 const calcDepFlags = (tree, resetRoot = true) => {
   if (resetRoot) {
-    tree.dev = false
-    tree.optional = false
-    tree.devOptional = false
-    tree.peer = false
+    tree.unsetDepFlags()
   }
-  const ret = depth({
-    tree,
-    visit: node => calcDepFlagsStep(node),
-    filter: node => node,
-    getChildren: (node, tree) =>
-      [...tree.edgesOut.values()].map(edge => edge.to),
-  })
-  return ret
-}
-
-const calcDepFlagsStep = (node) => {
-  // This rewalk is necessary to handle cases where devDep and optional
-  // or normal dependency graphs overlap deep in the dep graph.
-  // Since we're only walking through deps that are not already flagged
-  // as non-dev/non-optional, it's typically a very shallow traversal
-
-  node.extraneous = false
-  resetParents(node, 'extraneous')
-  resetParents(node, 'dev')
-  resetParents(node, 'peer')
-  resetParents(node, 'devOptional')
-  resetParents(node, 'optional')
-
-  // for links, map their hierarchy appropriately
-  if (node.isLink) {
-    // node.target can be null, we check to ensure it's not null before proceeding
-    if (node.target == null) {
-      return node
-    }
-    node.target.dev = node.dev
-    node.target.optional = node.optional
-    node.target.devOptional = node.devOptional
-    node.target.peer = node.peer
-    return calcDepFlagsStep(node.target)
-  }
-
-  node.edgesOut.forEach(({ peer, optional, dev, to }) => {
-    // if the dep is missing, then its flags are already maximally unset
-    if (!to) {
-      return
-    }
-    // everything with any kind of edge into it is not extraneous
-    to.extraneous = false
-
-    // If this is a peer edge, mark the target as peer
-    if (peer) {
-      to.peer = true
-    } else if (to.peer && !hasIncomingPeerEdge(to)) {
-      unsetFlag(to, 'peer')
-    }
 
-    // devOptional is the *overlap* of the dev and optional tree.
-    // however, for convenience and to save an extra rewalk, we leave
-    // it set when we are in *either* tree, and then omit it from the
-    // package-lock if either dev or optional are set.
-    const unsetDevOpt = !node.devOptional && !node.dev && !node.optional && !dev && !optional
+  const seen = new Set()
+  const queue = [tree]
 
-    // if we are not in the devOpt tree, then we're also not in
-    // either the dev or opt trees
-    const unsetDev = unsetDevOpt || !node.dev && !dev
-    const unsetOpt = unsetDevOpt || !node.optional && !optional
+  let node
+  while (node = queue.pop()) {
+    seen.add(node)
 
-    if (unsetDevOpt) {
-      unsetFlag(to, 'devOptional')
+    // Unset extraneous from all parents to avoid removal of children.
+    if (!node.extraneous) {
+      for (let n = node.resolveParent; n?.extraneous; n = n.resolveParent) {
+        n.extraneous = false
+      }
     }
 
-    if (unsetDev) {
-      unsetFlag(to, 'dev')
+    // for links, map their hierarchy appropriately
+    if (node.isLink) {
+      // node.target can be null, we check to ensure it's not null before proceeding
+      if (node.target == null) {
+        continue
+      }
+      node.target.dev = node.dev
+      node.target.optional = node.optional
+      node.target.devOptional = node.devOptional
+      node.target.peer = node.peer
+      node.target.extraneous = node.extraneous
+      queue.push(node.target)
+      continue
     }
 
-    if (unsetOpt) {
-      unsetFlag(to, 'optional')
-    }
-  })
-
-  return node
-}
-
-const hasIncomingPeerEdge = (node) => {
-  const target = node.isLink && node.target ? node.target : node
-  for (const edge of target.edgesIn) {
-    if (edge.type === 'peer') {
-      return true
+    for (const { peer, optional, dev, to } of node.edgesOut.values()) {
+      // if the dep is missing, then its flags are already maximally unset
+      if (!to) {
+        continue
+      }
+
+      let changed = false
+
+      // only optional peer dependencies should stay extraneous
+      if (to.extraneous && !node.extraneous && !(peer && optional)) {
+        to.extraneous = false
+        changed = true
+      }
+
+      if (to.dev && !node.dev && !dev) {
+        to.dev = false
+        changed = true
+      }
+
+      if (to.optional && !node.optional && !optional) {
+        to.optional = false
+        changed = true
+      }
+
+      // devOptional is the *overlap* of the dev and optional tree.
+      // A node may be depended on by separate dev and optional nodes.
+      // It SHOULD NOT be removed when pruning dev OR optional.
+      // It SHOULD be removed when pruning dev AND optional.
+      // We only unset here if a node is not dev AND not optional because
+      // if we did unset, it would prevent any overlap deeper in the tree.
+      // We correct this later by removing if dev OR optional is set.
+      if (to.devOptional && !node.devOptional && !node.dev && !node.optional && !dev && !optional) {
+        to.devOptional = false
+        changed = true
+      }
+
+      if (to.peer && !node.peer && !peer) {
+        to.peer = false
+        changed = true
+      }
+
+      if (changed) {
+        queue.push(to)
+      }
     }
   }
-  return false
-}
 
-const resetParents = (node, flag) => {
-  if (node[flag]) {
-    return
-  }
-
-  for (let p = node; p && (p === node || p[flag]); p = p.resolveParent) {
-    p[flag] = false
-  }
-}
-
-// typically a short walk, since it only traverses deps that have the flag set.
-const unsetFlag = (node, flag) => {
-  if (node[flag]) {
-    node[flag] = false
-    depth({
-      tree: node,
-      visit: node => {
-        node.extraneous = node[flag] = false
-        if (node.isLink && node.target) {
-          node.target.extraneous = node.target[flag] = false
-        }
-      },
-      getChildren: node => {
-        const children = []
-        const targetNode = node.isLink && node.target ? node.target : node
-        for (const edge of targetNode.edgesOut.values()) {
-          if (edge.to?.[flag]) {
-            // For the peer flag, only follow peer edges to unset the flag
-            // Don't propagate peer flag through prod/dev/optional edges
-            if (flag === 'peer') {
-              if (edge.type === 'peer') {
-                children.push(edge.to)
-              }
-            } else {
-              // For other flags, follow prod edges (and peer edges for non-peer flags)
-              if (edge.type === 'prod' || edge.type === 'peer') {
-                children.push(edge.to)
-              }
-            }
-          }
-        }
-        return children
-      },
-    })
+  // Remove incorrect devOptional flags now that we have walked all deps.
+  seen.delete(tree)
+  for (const node of seen.values()) {
+    if (node.devOptional && (node.dev || node.optional)) {
+      node.devOptional = false
+    }
   }
 }
 

package/lib/diff.js

@@ -11,9 +11,8 @@ const { existsSync } = require('node:fs')
 const ssri = require('ssri')
 
 class Diff {
-  constructor ({ actual, ideal, filterSet, shrinkwrapInflated, omit, omitted }) {
+  constructor ({ actual, ideal, filterSet, shrinkwrapInflated, omit }) {
     this.omit = omit
-    this.omitted = omitted
     this.filterSet = filterSet
     this.shrinkwrapInflated = shrinkwrapInflated
     this.children = []
@@ -39,7 +38,6 @@ class Diff {
     filterNodes = [],
     shrinkwrapInflated = new Set(),
     omit = new Set(),
-    omitted = new Set(),
   }) {
     // if there's a filterNode, then:
     // - get the path from the root to the filterNode.  The root or
@@ -98,28 +96,18 @@ class Diff {
     }
 
     return depth({
-      tree: new Diff({ actual, ideal, filterSet, shrinkwrapInflated, omit, omitted }),
+      tree: new Diff({ actual, ideal, filterSet, shrinkwrapInflated, omit }),
       getChildren,
       leave,
     })
   }
 }
 
-const getAction = ({ actual, ideal, omit, omitted }) => {
+const getAction = ({ actual, ideal }) => {
   if (!ideal) {
     return 'REMOVE'
   }
 
-  if (ideal.shouldOmit?.(omit)) {
-    omitted.add(ideal)
-
-    if (actual) {
-      return 'REMOVE'
-    }
-
-    return null
-  }
-
   // bundled meta-deps are copied over to the ideal tree when we visit it,
   // so they'll appear to be missing here.  There's no need to handle them
   // in the diff, though, because they'll be replaced at reify time anyway
@@ -199,7 +187,6 @@ const getChildren = diff => {
     filterSet,
     shrinkwrapInflated,
     omit,
-    omitted,
   } = diff
 
   // Note: we DON'T diff fsChildren themselves, because they are either
@@ -231,7 +218,6 @@ const getChildren = diff => {
       filterSet,
       shrinkwrapInflated,
       omit,
-      omitted,
     })
   }
 
@@ -251,13 +237,24 @@ const diffNode = ({
   filterSet,
   shrinkwrapInflated,
   omit,
-  omitted,
 }) => {
   if (filterSet.size && !(filterSet.has(ideal) || filterSet.has(actual))) {
     return
   }
 
-  const action = getAction({ actual, ideal, omit, omitted })
+  if (ideal?.shouldOmit?.(omit)) {
+    ideal.inert = true
+  }
+
+  // Treat inert nodes as undefined for the purposes of diffing.
+  if (ideal?.inert) {
+    ideal = undefined
+  }
+  if (!actual && !ideal) {
+    return
+  }
+
+  const action = getAction({ actual, ideal })
 
   // if it's a match, then get its children
   // otherwise, this is the child diff node
@@ -265,7 +262,7 @@ const diffNode = ({
     if (action === 'REMOVE') {
       removed.push(actual)
     }
-    children.push(new Diff({ actual, ideal, filterSet, shrinkwrapInflated, omit, omitted }))
+    children.push(new Diff({ actual, ideal, filterSet, shrinkwrapInflated, omit }))
   } else {
     unchanged.push(ideal)
     // !*! Weird dirty hack warning !*!
@@ -306,7 +303,6 @@ const diffNode = ({
       filterSet,
       shrinkwrapInflated,
       omit,
-      omitted,
     }))
   }
 }

package/lib/edge.js

@@ -276,9 +276,15 @@ class Edge {
       } else if (!this.satisfiedBy(this.#to)) {
         this.#error = 'INVALID'
       } else if (this.overrides && this.#to.edgesOut.size && OverrideSet.doOverrideSetsConflict(this.overrides, this.#to.overrides)) {
-        // Any inconsistency between the edge's override set and the target's override set is potentially problematic.
-        // But we only say the edge is in error if the override sets are plainly conflicting.
-        // Note that if the target doesn't have any dependencies of their own, then this inconsistency is irrelevant.
+        // Check for conflicts between the edge's override set and the target node's override set.
+        // This catches cases where different parts of the tree have genuinely incompatible
+        // version requirements for the same package.
+        // The improved conflict detection uses semantic comparison (checking for incompatible
+        // version ranges) rather than pure structural equality, avoiding false positives from:
+        // - Reference overrides ($syntax) that resolve to compatible versions
+        // - Peer dependencies with different but compatible override contexts
+        // Note: We only check if the target has dependencies (edgesOut.size > 0), since
+        // override conflicts are only relevant if the target has its own dependencies.
         this.#error = 'INVALID'
       } else {
         this.#error = 'OK'

package/lib/gather-dep-set.js

@@ -20,7 +20,7 @@ const gatherDepSet = (set, edgeFilter) => {
     }
   }
 
-  // now remove all nodes in the set that have a dependant outside the set
+  // now remove all nodes in the set that have a dependent outside the set
   // if any change is made, then re-check
   // continue until no changes made, or deps set evaporates fully.
   let changed = true

package/lib/node.js

@@ -101,7 +101,7 @@ class Node {
       global = false,
       dummy = false,
       sourceReference = null,
-      ideallyInert = false,
+      inert = false,
     } = options
     // this object gives querySelectorAll somewhere to stash context about a node
     // while processing a query
@@ -207,7 +207,7 @@ class Node {
       this.extraneous = false
     }
 
-    this.ideallyInert = ideallyInert
+    this.inert = inert
 
     this.edgesIn = new Set()
     this.edgesOut = new CaseInsensitiveMap()
@@ -248,7 +248,7 @@ class Node {
     this.fsParent = fsParent || null
 
     // see parent/root setters below.
-    // root is set to parent's root if we have a parent, otherwise if it's
+    // root is set to parent's root if we have a parent; otherwise, if it's
     // null, then it's set to the node itself.
     if (!parent && !fsParent) {
       this.root = root || null
@@ -832,7 +832,7 @@ class Node {
         edge.reload()
       }
     }
-    // reload all edgesOut where root doens't match, or is missing, since
+    // reload all edgesOut where root doesn't match, or is missing, since
     // it might not be missing in the new tree
     for (const edge of this.edgesOut.values()) {
       if (!edge.to || edge.to.root !== root) {
@@ -1268,7 +1268,7 @@ class Node {
   // with another by the same name (eg, to update or dedupe).
   // This does a couple of walks out on the node_modules tree, recursing
   // into child nodes.  However, as setting the parent is typically done
-  // with nodes that don't have have many children, and (deduped) package
+  // with nodes that don't have many children, and (deduped) package
   // trees tend to be broad rather than deep, it's not that bad.
   // The only walk that starts from the parent rather than this node is
   // limited by edge name.
@@ -1412,7 +1412,7 @@ class Node {
   }
 
   recalculateOutEdgesOverrides () {
-    // For each edge out propogate the new overrides through.
+    // For each edge out propagate the new overrides through.
     for (const edge of this.edgesOut.values()) {
       edge.reload(true)
       if (edge.to) {
@@ -1613,6 +1613,22 @@ class Node {
   [util.inspect.custom] () {
     return this.toJSON()
   }
+
+  resetDepFlags () {
+    this.extraneous = true
+    this.dev = true
+    this.optional = true
+    this.devOptional = true
+    this.peer = true
+  }
+
+  unsetDepFlags () {
+    this.extraneous = false
+    this.dev = false
+    this.optional = false
+    this.devOptional = false
+    this.peer = false
+  }
 }
 
 module.exports = Node