@npmcli/arborist 9.4.2 → 9.5.0

package/lib/arborist/build-ideal-tree.js

@@ -13,7 +13,6 @@ const { lstat, readlink } = require('node:fs/promises')
 const { depth } = require('treeverse')
 const { log, time } = require('proc-log')
 const { redact } = require('@npmcli/redact')
-const semver = require('semver')
 
 const {
   OK,
@@ -294,10 +293,6 @@ module.exports = cls => class IdealTreeBuilder extends cls {
           }).then(meta => Object.assign(root, { meta }))
         } else {
           return this.loadVirtual({ root })
-            .then(tree => {
-              this.#applyRootOverridesToWorkspaces(tree)
-              return tree
-            })
         }
       })
 
@@ -406,6 +401,7 @@ module.exports = cls => class IdealTreeBuilder extends cls {
         global: this.options.global,
         installLinks: this.installLinks,
         legacyPeerDeps: this.legacyPeerDeps,
+        loadOverrides: true,
         root,
       })
     }
@@ -450,6 +446,11 @@ module.exports = cls => class IdealTreeBuilder extends cls {
       const paths = await readdirScoped(nm).catch(() => [])
       for (const p of paths) {
         const name = p.replace(/\\/g, '/')
+        // Match loadActual behavior: hidden entries and retired scoped package
+        // folders are not installed global packages.
+        if (/^(@[^/]+\/)?\./.test(name)) {
+          continue
+        }
         const updateName = this[_updateNames].includes(name)
         if (this[_updateAll] || updateName) {
           if (updateName) {
@@ -913,8 +914,21 @@ This is a one-time fix-up, please be patient...
       // be forced to agree on a version of z.
       const required = new Set([edge.from])
       const parent = edge.peer ? virtualRoot : null
-      const dep = vrDep && vrDep.satisfies(edge) ? vrDep
-        : await this.#nodeFromEdge(edge, parent, null, required)
+      let dep = vrDep && vrDep.satisfies(edge) ? vrDep : null
+
+      // A peerOptional conflict can be resolved by finding an existing node in the tree that satisfies the edge, avoiding a registry fetch that may introduce an extraneous package. See npm/cli#9249.
+      // Skip the shortcut when the user has signaled an explicit re-fetch intent (npm update by name, explicit request, or audit fix), so we honor those signals rather than silently keeping the existing node.
+      const skipExistingShortcut = this[_updateNames].includes(edge.name)
+        || this.#explicitRequests.has(edge)
+        || (edge.to && this.auditReport?.isVulnerable(edge.to))
+      if (!dep && edge.type === 'peerOptional' && !skipExistingShortcut) {
+        dep = this.#findHoistableNode(
+          /* istanbul ignore next - resolveParent is always set for non-root nodes */
+          edge.from.resolveParent || edge.from, edge)
+      }
+      if (!dep) {
+        dep = await this.#nodeFromEdge(edge, parent, null, required)
+      }
 
       /* istanbul ignore next */
       debug(() => {
@@ -950,7 +964,7 @@ This is a one-time fix-up, please be patient...
         tree: pd,
         getChildren: pd => pd.children,
         visit: pd => {
-          const { placed, edge, canPlace: cpd } = pd
+          const { placed, edge, canPlace: cpd, parent } = pd
           // if we didn't place anything, nothing to do here
           if (!placed) {
             return
@@ -1011,8 +1025,7 @@ This is a one-time fix-up, please be patient...
             return
           }
 
-          // lastly, also check for the missing deps of the node we placed,
-          // and any holes created by pruning out conflicted peer sets.
+          // lastly, also check for the missing deps of the node we placed, and any holes created by pruning out conflicted peer sets.
           this.#depsQueue.push(placed)
           for (const dep of pd.needEvaluation) {
             this.#depsSeen.delete(dep)
@@ -1020,16 +1033,14 @@ This is a one-time fix-up, please be patient...
           }
 
           // pre-fetch any problem edges, since we'll need these soon
-          // if it fails at this point, though, don't worry because it
-          // may well be an optional dep that has gone missing.  it'll
-          // fail later anyway.
+          // if it fails at this point, though, don't worry because it may well be an optional dep that has gone missing
+          // it'll fail later anyway
           for (const e of this.#problemEdges(placed)) {
-            // XXX This is somehow load bearing.  This makes tests that print
-            // the ideal tree of a tree with tarball dependencies fail. This
-            // can't be changed or removed till we figure out why
+            // XXX This is somehow load bearing.  This makes tests that print the ideal tree of a tree with tarball dependencies fail
+            // This can't be changed or removed till we figure out why
             // The test is named "tarball deps with transitive tarball deps"
             promises.push(() =>
-              this.#fetchManifest(npa.resolve(e.name, e.spec, fromPath(placed, e)))
+              this.#fetchManifest(npa.resolve(e.name, e.spec, fromPath(placed, e)), parent)
                 .catch(() => null)
             )
           }
@@ -1047,26 +1058,36 @@ This is a one-time fix-up, please be patient...
     return this.#buildDepStep()
   }
 
-  // loads a node from an edge, and then loads its peer deps (and their
-  // peer deps, on down the line) into a virtual root parent.
+  // BFS descendants of `root` for a node satisfying `edge`.
+  // Prefers nodes closer to root.  Skips bundled nodes.
+  #findHoistableNode (root, edge) {
+    const queue = [...root.children.values()]
+    while (queue.length) {
+      const node = queue.shift()
+      if (node.name === edge.name
+        && !node.inDepBundle
+        && node.satisfies(edge)) {
+        return node
+      }
+      for (const child of node.children.values()) {
+        queue.push(child)
+      }
+    }
+    return null
+  }
+
+  // loads a node from an edge, and then loads its peer deps (and their peer deps, on down the line) into a virtual root parent.
   async #nodeFromEdge (edge, parent_, secondEdge, required) {
-    // create a virtual root node with the same deps as the node that
-    // is requesting this one, so that we can get all the peer deps in
-    // a context where they're likely to be resolvable.
-    // Note that the virtual root will also have virtual copies of the
-    // targets of any child Links, so that they resolve appropriately.
+    // create a virtual root node with the same deps as the node that is requesting this one, so that we can get all the peer deps in a context where they're likely to be resolvable.
+    // Note that the virtual root will also have virtual copies of the targets of any child Links, so that they resolve appropriately.
     const parent = parent_ || this.#virtualRoot(edge.from)
 
     const spec = npa.resolve(edge.name, edge.spec, edge.from.path)
     const first = await this.#nodeFromSpec(edge.name, spec, parent, edge)
 
-    // we might have a case where the parent has a peer dependency on
-    // `foo@*` which resolves to v2, but another dep in the set has a
-    // peerDependency on `foo@1`.  In that case, if we force it to be v2,
-    // we're unnecessarily triggering an ERESOLVE.
-    // If we have a second edge to worry about, and it's not satisfied
-    // by the first node, try a second and see if that satisfies the
-    // original edge here.
+    // we might have a case where the parent has a peer dependency on `foo@*` which resolves to v2, but another dep in the set has a peerDependency on `foo@1`.
+    // In that case, if we force it to be v2, we're unnecessarily triggering an ERESOLVE.
+    // If we have a second edge to worry about, and it's not satisfied by the first node, try a second and see if that satisfies the original edge here.
     const spec2 = secondEdge && npa.resolve(
       edge.name,
       secondEdge.spec,
@@ -1210,11 +1231,12 @@ This is a one-time fix-up, please be patient...
     return problems
   }
 
-  async #fetchManifest (spec) {
+  async #fetchManifest (spec, parent) {
     const options = {
       ...this.options,
       avoid: this.#avoidRange(spec.name),
       fullMetadata: true,
+      _isRoot: parent?.isProjectRoot || parent?.isWorkspace,
     }
     // get the intended spec and stored metadata from yarn.lock file,
     // if available and valid.
@@ -1231,10 +1253,8 @@ This is a one-time fix-up, please be patient...
   }
 
   async #nodeFromSpec (name, spec, parent, edge) {
-    // pacote will slap integrity on its options, so we have to clone
-    // the object so it doesn't get mutated.
-    // Don't bother to load the manifest for link deps, because the target
-    // might be within another package that doesn't exist yet.
+    // pacote will slap integrity on its options, so we have to clone the object so it doesn't get mutated.
+    // Don't bother to load the manifest for link deps, because the target might be within another package that doesn't exist yet.
     const { installLinks, legacyPeerDeps } = this
     const isWorkspace = this.idealTree.workspaces && this.idealTree.workspaces.has(spec.name)
 
@@ -1287,7 +1307,7 @@ This is a one-time fix-up, please be patient...
 
     // spec isn't a directory, and either isn't a workspace or the workspace we have
     // doesn't satisfy the edge. try to fetch a manifest and build a node from that.
-    return this.#fetchManifest(spec)
+    return this.#fetchManifest(spec, parent)
       .then(pkg => new Node({ name, pkg, parent, installLinks, legacyPeerDeps }), error => {
         error.requiredBy = edge.from.location || '.'
 
@@ -1519,32 +1539,6 @@ This is a one-time fix-up, please be patient...
     timeEnd()
   }
 
-  #applyRootOverridesToWorkspaces (tree) {
-    const rootOverrides = tree.root.package.overrides || {}
-
-    for (const node of tree.root.inventory.values()) {
-      if (!node.isWorkspace) {
-        continue
-      }
-
-      for (const depName of Object.keys(rootOverrides)) {
-        const edge = node.edgesOut.get(depName)
-        const rootNode = tree.root.children.get(depName)
-
-        // safely skip if either edge or rootNode doesn't exist yet
-        if (!edge || !rootNode) {
-          continue
-        }
-
-        const resolvedRootVersion = rootNode.package.version
-        if (!semver.satisfies(resolvedRootVersion, edge.spec)) {
-          edge.detach()
-          node.children.delete(depName)
-        }
-      }
-    }
-  }
-
   #idealTreePrune () {
     for (const node of this.idealTree.inventory.values()) {
       if (node.extraneous) {

package/lib/arborist/index.js

@@ -288,6 +288,16 @@ class Arborist extends Base {
     return ret
   }
 
+  // Build an ideal tree (or reuse an already-built one) and return the
+  // resulting lockfile contents as a string, without writing to disk.
+  // Useful for callers that want to inspect, diff, or store a lockfile
+  // somewhere other than the project's `package-lock.json`.
+  async lockfileString (options = {}) {
+    await this.buildIdealTree(options)
+
+    return this.idealTree.meta.toString(options)
+  }
+
   async dedupe (options = {}) {
     // allow the user to set options on the ctor as well.
     // XXX: deprecate separate method options objects.

package/lib/arborist/isolated-reifier.js

@@ -113,7 +113,11 @@ module.exports = cls => class IsolatedReifier extends cls {
       })
       // local `file:` deps are in fsChildren but are not workspaces.
       // they are already handled as workspace-like proxies above and should not go through the external/store extraction path.
-      if (!next.isProjectRoot && !next.isWorkspace && !next.inert && !idealTree.fsChildren.has(next) && !idealTree.fsChildren.has(next.target)) {
+      // Links with file: resolved paths (from `npm link`) should also be treated as local dependencies and symlinked directly instead of being extracted into the store.
+      const isLocalFileDep = next.isLink && next.resolved?.startsWith('file:')
+      if (isLocalFileDep && !idealTree.fsChildren.has(next) && !idealTree.fsChildren.has(next.target)) {
+        this.idealGraph.workspaces.push(await this.#workspaceProxy(next.target))
+      } else if (!next.isProjectRoot && !next.isWorkspace && !next.inert && !idealTree.fsChildren.has(next) && !idealTree.fsChildren.has(next.target)) {
         this.idealGraph.external.push(await this.#externalProxy(next))
       }
     }
@@ -157,6 +161,7 @@ module.exports = cls => class IsolatedReifier extends cls {
         ...this.options,
         resolved: node.resolved,
         integrity: node.integrity,
+        // TODO _isRoot
       })
       const Arborist = this.constructor
       const arb = new Arborist({ ...this.options, path: dir })

package/lib/arborist/reify.js

@@ -11,7 +11,7 @@ const { depth: dfwalk } = require('treeverse')
 const { dirname, resolve, relative, join, sep } = require('node:path')
 const { log, time } = require('proc-log')
 const { existsSync } = require('node:fs')
-const { lstat, mkdir, readdir, rm, symlink } = require('node:fs/promises')
+const { lstat, mkdir, readdir, readlink, rm, symlink } = require('node:fs/promises')
 const { moveFile } = require('@npmcli/fs')
 const { subset, intersects } = require('semver')
 const { walkUp } = require('walk-up-path')
@@ -126,7 +126,11 @@ module.exports = cls => class Reifier extends cls {
     await this[_diffTrees]()
     await this.#reifyPackages()
     if (linked) {
-      await this.#cleanOrphanedStoreEntries()
+      // The sweep mutates node_modules on disk, so skip it for dry runs and lockfile-only installs (those modes also short-circuit #reifyPackages).
+      // The sweep itself scopes to in-filter workspaces when a filter is active, so it's safe to run for filtered installs too.
+      if (!this.options.dryRun && !this.options.packageLockOnly) {
+        await this.#cleanOrphanedStoreEntries()
+      }
       // swap back in the idealTree
       // so that the lockfile is preserved
       this.idealTree = oldTree
@@ -737,6 +741,7 @@ module.exports = cls => class Reifier extends cls {
         ...this.options,
         resolved: node.resolved,
         integrity: node.integrity,
+        _isRoot: node.parent?.isProjectRoot || node.parent?.isWorkspace,
       })
       // store nodes don't use Node class so node.package doesn't get updated
       if (node.isInStore) {
@@ -1320,35 +1325,175 @@ module.exports = cls => class Reifier extends cls {
 
   // After a linked install, scan node_modules/.store/ and remove any directories that are not referenced by the current ideal tree.
   // Store entries become orphaned when dependencies are updated or removed, because the diff never sees the old store keys.
+  // Then sweep the top-level node_modules/ for orphaned symlinks (e.g. an uninstalled dep whose store entry was just removed) so we don't leave dangling links.
   async #cleanOrphanedStoreEntries () {
-    const storeDir = resolve(this.path, 'node_modules', '.store')
+    const nmDir = resolve(this.path, 'node_modules')
+    const storeDir = resolve(nmDir, '.store')
+
     let entries
     try {
       entries = await readdir(storeDir)
     } catch {
-      return
+      entries = null
     }
 
-    // Collect valid store keys from the isolated ideal tree (location: node_modules/.store/{key}/node_modules/{pkg})
+    // Collect valid store keys and valid top-level links per node_modules directory.
+    // Store entries have location node_modules/.store/{key}/node_modules/{pkg}.
+    // Top-level links have location {prefix}/node_modules/{pkg} or {prefix}/node_modules/@scope/{pkg}, where {prefix} is empty for the root project and the workspace's localLocation for workspace deps.
+    // Locations are normalized to forward slashes here because IsolatedNode/IsolatedLink locations are built with path.join, which uses backslashes on Windows.
     const validKeys = new Set()
+    const nmDirs = new Map()
+    const NM_PREFIX = 'node_modules/'
+    const STORE_MARKER = '/.store/'
     for (const child of this.idealTree.children.values()) {
+      const loc = child.location.replace(/\\/g, '/')
       if (child.isInStore) {
-        const key = child.location.split(sep)[2]
+        const key = loc.split('/')[2]
         validKeys.add(key)
+        continue
+      }
+      if (!child.isLink) {
+        continue
+      }
+      const nmIdx = loc.lastIndexOf(NM_PREFIX)
+      if (nmIdx === -1 || loc.includes(STORE_MARKER)) {
+        continue
+      }
+      const prefix = loc.slice(0, nmIdx)
+      const dir = resolve(this.path, prefix, 'node_modules')
+      const rest = loc.slice(nmIdx + NM_PREFIX.length)
+      let entry
+      if (rest.startsWith('@')) {
+        const [scope, name] = rest.split('/')
+        entry = `${scope}${sep}${name}`
+      } else {
+        entry = rest.split('/')[0]
+      }
+      let set = nmDirs.get(dir)
+      if (!set) {
+        set = new Set()
+        nmDirs.set(dir, set)
+      }
+      set.add(entry)
+    }
+
+    // Determine which node_modules directories to sweep.
+    // For an unfiltered install, sweep the project root and every workspace's node_modules even if no top-level links remain (e.g. last dep was just uninstalled).
+    // For a filtered install (npm install -w <ws>), restrict the sweep to the in-scope workspaces so out-of-scope workspaces are left untouched, mirroring what the diff would do.
+    // When --include-workspace-root is set, the filter scope pulls in root deps too, so the root node_modules is included in the sweep.
+    const filteredNames = this.options.workspaces
+    const isFiltered = Array.isArray(filteredNames) && filteredNames.length > 0
+    if (isFiltered) {
+      const allowedDirs = new Set()
+      for (const ws of this.idealTree.fsChildren) {
+        if (filteredNames.includes(ws.packageName) || filteredNames.includes(ws.name)) {
+          allowedDirs.add(resolve(ws.path, 'node_modules'))
+        }
+      }
+      if (this.options.includeWorkspaceRoot) {
+        allowedDirs.add(nmDir)
+      }
+      for (const dir of [...nmDirs.keys()]) {
+        if (!allowedDirs.has(dir)) {
+          nmDirs.delete(dir)
+        }
+      }
+      for (const dir of allowedDirs) {
+        if (!nmDirs.has(dir)) {
+          nmDirs.set(dir, new Set())
+        }
+      }
+    } else {
+      if (!nmDirs.has(nmDir)) {
+        nmDirs.set(nmDir, new Set())
+      }
+      for (const ws of this.idealTree.fsChildren) {
+        const wsNmDir = resolve(ws.path, 'node_modules')
+        if (!nmDirs.has(wsNmDir)) {
+          nmDirs.set(wsNmDir, new Set())
+        }
+      }
+    }
+
+    if (entries) {
+      const orphaned = entries.filter(e => !validKeys.has(e))
+      if (orphaned.length) {
+        log.silly('reify', 'cleaning orphaned store entries', orphaned)
+        await promiseAllRejectLate(
+          orphaned.map(e =>
+            rm(resolve(storeDir, e), { recursive: true, force: true })
+              .catch(/* istanbul ignore next -- rm with force rarely fails */
+                er => log.warn('cleanup', `Failed to remove orphaned store entry ${e}`, er))
+          )
+        )
+      }
+    }
+
+    for (const [dir, valid] of nmDirs) {
+      await this.#cleanOrphanedTopLevelLinks(dir, valid)
+    }
+  }
+
+  // Remove node_modules/ entries that aren't represented in the ideal tree.
+  // Run for the project root and each workspace's node_modules.
+  // The linked diff path can't see these because #buildLinkedActualForDiff derives the actual tree from the ideal, so removed deps are never compared.
+  // Only symlinks whose target resolves inside the project root are removed — that covers store links (node_modules/.store/...) and workspace self-links (e.g. node_modules/<ws> -> ../packages/<ws>) that npm itself created.
+  // Symlinks pointing outside the project (e.g. `npm link foo` without --save targeting the global prefix, or hand-made `ln -s` to an external path) and real directories are preserved.
+  async #cleanOrphanedTopLevelLinks (nmDir, validTopLevel) {
+    const projectPrefix = resolve(this.path) + sep
+    let dirents
+    try {
+      dirents = await readdir(nmDir, { withFileTypes: true })
+    } catch {
+      return
+    }
+
+    const isOurOrphan = async (linkPath) => {
+      let target
+      try {
+        target = await readlink(linkPath)
+      } catch {
+        /* istanbul ignore next -- readlink of an entry we just listed as a symlink should not fail */
+        return false
+      }
+      return resolve(dirname(linkPath), target).startsWith(projectPrefix)
+    }
+
+    const orphaned = []
+    for (const ent of dirents) {
+      // skip npm-managed entries (.bin, .store, .package-lock.json, etc)
+      if (ent.name.startsWith('.')) {
+        continue
+      }
+      if (ent.name.startsWith('@')) {
+        let scoped
+        try {
+          scoped = await readdir(resolve(nmDir, ent.name), { withFileTypes: true })
+        } catch {
+          /* istanbul ignore next -- readdir of an entry we just listed should not fail */
+          continue
+        }
+        for (const pkgEnt of scoped) {
+          const key = `${ent.name}${sep}${pkgEnt.name}`
+          if (!validTopLevel.has(key) && pkgEnt.isSymbolicLink() && await isOurOrphan(resolve(nmDir, key))) {
+            orphaned.push(key)
+          }
+        }
+      } else if (!validTopLevel.has(ent.name) && ent.isSymbolicLink() && await isOurOrphan(resolve(nmDir, ent.name))) {
+        orphaned.push(ent.name)
       }
     }
 
-    const orphaned = entries.filter(e => !validKeys.has(e))
     if (!orphaned.length) {
       return
     }
 
-    log.silly('reify', 'cleaning orphaned store entries', orphaned)
+    log.silly('reify', 'cleaning orphaned top-level links', orphaned)
     await promiseAllRejectLate(
-      orphaned.map(e =>
-        rm(resolve(storeDir, e), { recursive: true, force: true })
+      orphaned.map(name =>
+        rm(resolve(nmDir, name), { recursive: true, force: true })
           .catch(/* istanbul ignore next -- rm with force rarely fails */
-            er => log.warn('cleanup', `Failed to remove orphaned store entry ${e}`, er))
+            er => log.warn('cleanup', `Failed to remove orphaned link ${name}`, er))
       )
     )
   }

package/lib/link.js

@@ -109,6 +109,14 @@ class Link extends Node {
   // so this is a no-op
   [_loadDeps] () {}
 
+  // When a Link receives overrides (via edgesIn), forward them to the target node which holds the actual edgesOut.
+  // Without this, overrides stop at the Link and never reach the target's dependency edges.
+  recalculateOutEdgesOverrides () {
+    if (this.target) {
+      this.target.updateOverridesEdgeInAdded(this.overrides)
+    }
+  }
+
   // links can't have children, only their targets can
   // fix it to an empty list so that we can still call
   // things that iterate over them, just as a no-op

package/lib/optional-set.js

@@ -26,7 +26,7 @@ const optionalSet = node => {
 
   // now that we've hit the boundary, gather the rest of the nodes in
   // the optional section that don't have dependents outside the set.
-  return gatherDepSet(set, edge => !set.has(edge.to))
+  return gatherDepSet(set, edge => !set.has(edge.to) && !edge.from?.inert)
 }
 
 module.exports = optionalSet

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@npmcli/arborist",
-  "version": "9.4.2",
+  "version": "9.5.0",
   "description": "Manage node_modules trees",
   "dependencies": {
     "@gar/promise-retry": "^1.0.0",