@npmcli/arborist 9.1.5 → 9.1.7

package/README.md

@@ -6,7 +6,7 @@
 
 Inspect and manage `node_modules` trees.
 
-![a tree with the word ARBORIST superimposed on it](https://raw.githubusercontent.com/npm/arborist/main/docs/logo.svg?sanitize=true)
+![a tree with the word ARBORIST superimposed on it](https://raw.githubusercontent.com/npm/cli/latest/workspaces/arborist/docs/logo.svg?sanitize=true)
 
 There's more documentation [in the docs
 folder](https://github.com/npm/cli/tree/latest/workspaces/arborist/docs).

package/bin/index.js

@@ -37,7 +37,7 @@ ${message && '\n' + message + '\n'}
 
   Additionally:
 
-  * --loglevel=warn|--quiet will supppress the printing of package trees
+  * --loglevel=warn|--quiet will suppress the printing of package trees
   * --logfile <file|bool> will output logs to a file
   * --timing will show timing information
   * Instead of 'npm install <pkg>', use 'arborist reify --add=<pkg>'.

package/lib/arborist/build-ideal-tree.js

@@ -42,7 +42,6 @@ const _flagsSuspect = Symbol.for('flagsSuspect')
 const _setWorkspaces = Symbol.for('setWorkspaces')
 const _updateNames = Symbol.for('updateNames')
 const _resolvedAdd = Symbol.for('resolvedAdd')
-const _usePackageLock = Symbol.for('usePackageLock')
 const _rpcache = Symbol.for('realpathCache')
 const _stcache = Symbol.for('statCache')
 
@@ -101,39 +100,28 @@ module.exports = cls => class IdealTreeBuilder extends cls {
   constructor (options) {
     super(options)
 
-    // normalize trailing slash
-    const registry = options.registry || 'https://registry.npmjs.org'
-    options.registry = this.registry = registry.replace(/\/+$/, '') + '/'
-
     const {
       follow = false,
       installStrategy = 'hoisted',
-      idealTree = null,
-      installLinks = false,
-      legacyPeerDeps = false,
-      packageLock = true,
       strictPeerDeps = false,
-      workspaces,
       global,
     } = options
 
     this.#strictPeerDeps = !!strictPeerDeps
 
-    this.idealTree = idealTree
-    this.installLinks = installLinks
-    this.legacyPeerDeps = legacyPeerDeps
-
-    this[_usePackageLock] = packageLock
     this.#installStrategy = global ? 'shallow' : installStrategy
     this.#follow = !!follow
 
-    if (workspaces?.length && global) {
-      throw new Error('Cannot operate on workspaces in global mode')
-    }
-
     this[_updateAll] = false
     this[_updateNames] = []
     this[_resolvedAdd] = []
+
+    // caches for cached realpath calls
+    const cwd = process.cwd()
+    // assume that the cwd is real enough for our purposes
+    this[_rpcache] = new Map([[cwd, cwd]])
+    this[_stcache] = new Map()
+    this[_flagsSuspect] = false
   }
 
   get explicitRequests () {
@@ -192,7 +180,7 @@ module.exports = cls => class IdealTreeBuilder extends cls {
   }
 
   async #checkEngineAndPlatform () {
-    const { engineStrict, npmVersion, nodeVersion, omit = [] } = this.options
+    const { engineStrict, npmVersion, nodeVersion, omit = [], cpu, os, libc } = this.options
     const omitSet = new Set(omit)
 
     for (const node of this.idealTree.inventory.values()) {
@@ -214,6 +202,19 @@ module.exports = cls => class IdealTreeBuilder extends cls {
         }
         checkPlatform(node.package, this.options.force)
       }
+      if (node.optional && !node.inert) {
+        // Mark any optional packages we can't install as inert.
+        // We ignore the --force and --engine-strict flags.
+        try {
+          checkEngine(node.package, npmVersion, nodeVersion, false)
+          checkPlatform(node.package, false, { cpu, os, libc })
+        } catch (error) {
+          const set = optionalSet(node)
+          for (const node of set) {
+            node.inert = true
+          }
+        }
+      }
     }
   }
 
@@ -285,7 +286,7 @@ module.exports = cls => class IdealTreeBuilder extends cls {
       .then(root => {
         if (this.options.global) {
           return root
-        } else if (!this[_usePackageLock] || this[_updateAll]) {
+        } else if (!this.options.usePackageLock || this[_updateAll]) {
           return Shrinkwrap.reset({
             path: this.path,
             lockfileVersion: this.options.lockfileVersion,
@@ -324,7 +325,7 @@ module.exports = cls => class IdealTreeBuilder extends cls {
       })
 
       .then(tree => {
-        // search the virtual tree for invalid edges, if any are found add their source to
+        // search the virtual tree for missing/invalid edges, if any are found add their source to
         // the depsQueue so that we'll fix it later
         depth({
           tree,
@@ -338,7 +339,7 @@ module.exports = cls => class IdealTreeBuilder extends cls {
           filter: node => node,
           visit: node => {
             for (const edge of node.edgesOut.values()) {
-              if (!edge.valid) {
+              if ((!edge.to && edge.type !== 'peerOptional') || !edge.valid) {
                 this.#depsQueue.push(node)
                 break // no need to continue the loop after the first hit
               }
@@ -741,6 +742,7 @@ This is a one-time fix-up, please be patient...
 
     // have to re-calc dep flags, because the nodes don't have edges
     // until their packages get assigned, so everything looks extraneous
+    resetDepFlags(this.idealTree)
     calcDepFlags(this.idealTree)
 
     // yes, yes, this isn't the "original" version, but now that it's been
@@ -811,7 +813,7 @@ This is a one-time fix-up, please be patient...
       node !== this.idealTree &&
       node.resolved &&
       (hasBundle || hasShrinkwrap) &&
-      !node.ideallyInert
+      !node.inert
     if (crackOpen) {
       const Arborist = this.constructor
       const opt = { ...this.options }
@@ -1011,7 +1013,7 @@ This is a one-time fix-up, please be patient...
           }
 
           // pre-fetch any problem edges, since we'll need these soon
-          // if it fails at this point, though, dont' worry because it
+          // if it fails at this point, though, don't worry because it
           // may well be an optional dep that has gone missing.  it'll
           // fail later anyway.
           for (const e of this.#problemEdges(placed)) {
@@ -1067,7 +1069,7 @@ This is a one-time fix-up, please be patient...
       ? await this.#nodeFromSpec(edge.name, spec2, parent, secondEdge)
       : null
 
-    // pick the second one if they're both happy with that, otherwise first
+    // pick the second one if they're both happy with that; otherwise, first
     const node = second && edge.valid ? second : first
     // ensure the one we want is the one that's placed
     node.parent = parent
@@ -1217,7 +1219,7 @@ This is a one-time fix-up, please be patient...
     }
   }
 
-  #nodeFromSpec (name, spec, parent, edge) {
+  async #nodeFromSpec (name, spec, parent, edge) {
     // pacote will slap integrity on its options, so we have to clone
     // the object so it doesn't get mutated.
     // Don't bother to load the manifest for link deps, because the target
@@ -1246,7 +1248,13 @@ This is a one-time fix-up, please be patient...
     // Decide whether to link or copy the dependency
     const shouldLink = (isWorkspace || isProjectInternalFileSpec || !installLinks) && !isTransitiveFileDep
     if (spec.type === 'directory' && shouldLink) {
-      return this.#linkFromSpec(name, spec, parent, edge)
+      const realpath = spec.fetchSpec
+      const { content: pkg } = await PackageJson.normalize(realpath).catch(() => {
+        return { content: {} }
+      })
+      const link = new Link({ name, parent, realpath, pkg, installLinks, legacyPeerDeps })
+      this.#linkNodes.add(link)
+      return link
     }
 
     // if the spec matches a workspace name, then see if the workspace node will satisfy the edge. if it does, we return the workspace node to make sure it takes priority.
@@ -1274,7 +1282,7 @@ This is a one-time fix-up, please be patient...
 
         // failed to load the spec, either because of enotarget or
         // fetch failure of some other sort.  save it so we can verify
-        // later that it's optional, otherwise the error is fatal.
+        // later that it's optional; otherwise, the error is fatal.
         const n = new Node({
           name,
           parent,
@@ -1287,17 +1295,6 @@ This is a one-time fix-up, please be patient...
       })
   }
 
-  async #linkFromSpec (name, spec, parent) {
-    const realpath = spec.fetchSpec
-    const { installLinks, legacyPeerDeps } = this
-    const { content: pkg } = await PackageJson.normalize(realpath).catch(() => {
-      return { content: {} }
-    })
-    const link = new Link({ name, parent, realpath, pkg, installLinks, legacyPeerDeps })
-    this.#linkNodes.add(link)
-    return link
-  }
-
   // load all peer deps and meta-peer deps into the node's parent
   // At the end of this, the node's peer-type outward edges are all
   // resolved, and so are all of theirs, but other dep types are not.
@@ -1431,7 +1428,8 @@ This is a one-time fix-up, please be patient...
   // - if a path under an existing node, then assign that as the fsParent,
   //   and add it to the _depsQueue
   //
-  // call buildDepStep if anything was added to the queue, otherwise we're done
+  // call buildDepStep if anything was added to the queue; otherwise, we're done
+  // XXX load-virtual also has a #resolveLinks, is there overlap?
   #resolveLinks () {
     for (const link of this.#linkNodes) {
       this.#linkNodes.delete(link)
@@ -1495,11 +1493,7 @@ This is a one-time fix-up, please be patient...
     } else {
       // otherwise just unset all the flags on the root node
       // since they will sometimes have the default value
-      this.idealTree.extraneous = false
-      this.idealTree.dev = false
-      this.idealTree.optional = false
-      this.idealTree.devOptional = false
-      this.idealTree.peer = false
+      this.idealTree.unsetDepFlags()
     }
 
     // at this point, any node marked as extraneous should be pruned.
@@ -1542,12 +1536,7 @@ This is a one-time fix-up, please be patient...
 
   #idealTreePrune () {
     for (const node of this.idealTree.inventory.values()) {
-      // optional peer dependencies are meant to be added to the tree
-      // through an explicit required dependency (most commonly in the
-      // root package.json), at which point they won't be optional so
-      // any dependencies still marked as both optional and peer at
-      // this point can be pruned as a special kind of extraneous
-      if (node.extraneous || (node.peer && node.optional)) {
+      if (node.extraneous) {
         node.parent = null
       }
     }
@@ -1561,7 +1550,7 @@ This is a one-time fix-up, please be patient...
 
       const set = optionalSet(node)
       for (const node of set) {
-        node.ideallyInert = true
+        node.inert = true
       }
     }
   }
@@ -1582,7 +1571,7 @@ This is a one-time fix-up, please be patient...
           node.parent !== null
           && !node.isProjectRoot
           && !excludeNodes.has(node)
-          && !node.ideallyInert
+          && !node.inert
         ) {
           this[_addNodeToTrashList](node)
         }

package/lib/arborist/index.js

@@ -68,6 +68,34 @@ class Arborist extends Base {
   constructor (options = {}) {
     const timeEnd = time.start('arborist:ctor')
     super(options)
+
+    // normalize trailing slash
+    const registry = options.registry || 'https://registry.npmjs.org'
+    options.registry = this.registry = registry.replace(/(?<!\/)\/+$/, '') + '/'
+
+    // TODO as we consolidate constructors it's more apparent that we are not parsing options and using this.options consistently
+    const {
+      actualTree,
+      global,
+      idealTree = null,
+      installLinks = false,
+      legacyPeerDeps = false,
+      virtualTree,
+      workspaces,
+    } = options
+
+    if (workspaces?.length && global) {
+      throw new Error('Cannot operate on workspaces in global mode')
+    }
+
+    // the tree of nodes on disk
+    this.actualTree = actualTree
+    this.idealTree = idealTree
+    this.installLinks = installLinks
+    this.legacyPeerDeps = legacyPeerDeps
+    // the virtual tree we load from a shrinkwrap
+    this.virtualTree = virtualTree
+
     this.options = {
       nodeVersion: process.version,
       ...options,
@@ -88,6 +116,7 @@ class Arborist extends Base {
       replaceRegistryHost: options.replaceRegistryHost,
       savePrefix: 'savePrefix' in options ? options.savePrefix : '^',
       scriptShell: options.scriptShell,
+      usePackageLock: 'packageLock' in options ? options.packageLock : true,
       workspaces: options.workspaces || [],
       workspacesEnabled: options.workspacesEnabled !== false,
     }
@@ -104,6 +133,7 @@ class Arborist extends Base {
     this.cache = resolve(this.options.cache)
     this.diff = null
     this.path = resolve(this.options.path)
+    this.scriptsRun = new Set()
     timeEnd()
   }
 

package/lib/arborist/isolated-reifier.js

@@ -1,6 +1,5 @@
 const _makeIdealGraph = Symbol('makeIdealGraph')
 const _createIsolatedTree = Symbol.for('createIsolatedTree')
-const _createBundledTree = Symbol('createBundledTree')
 const { mkdirSync } = require('node:fs')
 const pacote = require('pacote')
 const { join } = require('node:path')
@@ -81,7 +80,7 @@ module.exports = cls => class IsolatedReifier extends cls {
         }
         queue.push(e.to)
       })
-      if (!next.isProjectRoot && !next.isWorkspace && !next.ideallyInert) {
+      if (!next.isProjectRoot && !next.isWorkspace && !next.inert) {
         root.external.push(await this.externalProxyMemo(next))
       }
     }
@@ -140,15 +139,15 @@ module.exports = cls => class IsolatedReifier extends cls {
 
   async assignCommonProperties (node, result) {
     function validEdgesOut (node) {
-      return [...node.edgesOut.values()].filter(e => e.to && e.to.target && !(node.package.bundledDepenedencies || node.package.bundleDependencies || []).includes(e.to.name))
+      return [...node.edgesOut.values()].filter(e => e.to && e.to.target && !(node.package.bundledDependencies || node.package.bundleDependencies || []).includes(e.to.name))
     }
     const edges = validEdgesOut(node)
     const optionalDeps = edges.filter(e => e.optional).map(e => e.to.target)
     const nonOptionalDeps = edges.filter(e => !e.optional).map(e => e.to.target)
 
     result.localDependencies = await Promise.all(nonOptionalDeps.filter(n => n.isWorkspace).map(this.workspaceProxyMemo))
-    result.externalDependencies = await Promise.all(nonOptionalDeps.filter(n => !n.isWorkspace && !n.ideallyInert).map(this.externalProxyMemo))
-    result.externalOptionalDependencies = await Promise.all(optionalDeps.filter(n => !n.ideallyInert).map(this.externalProxyMemo))
+    result.externalDependencies = await Promise.all(nonOptionalDeps.filter(n => !n.isWorkspace && !n.inert).map(this.externalProxyMemo))
+    result.externalOptionalDependencies = await Promise.all(optionalDeps.filter(n => !n.inert).map(this.externalProxyMemo))
     result.dependencies = [
       ...result.externalDependencies,
       ...result.localDependencies,
@@ -162,7 +161,7 @@ module.exports = cls => class IsolatedReifier extends cls {
     result.hasInstallScript = node.hasInstallScript
   }
 
-  async [_createBundledTree] () {
+  async #createBundledTree () {
     // TODO: make sure that idealTree object exists
     const idealTree = this.idealTree
     // TODO: test workspaces having bundled deps
@@ -217,7 +216,7 @@ module.exports = cls => class IsolatedReifier extends cls {
 
     const proxiedIdealTree = this.idealGraph
 
-    const bundledTree = await this[_createBundledTree]()
+    const bundledTree = await this.#createBundledTree()
 
     const treeHash = (startNode) => {
       // generate short hash based on the dependency tree

package/lib/arborist/load-actual.js

@@ -36,24 +36,11 @@ module.exports = cls => class ActualLoader extends cls {
   // We don't do fsParent as a magic getter/setter, because it'd be too costly
   // to keep up to date along the walk.
   // And, we know that it can ONLY be relevant when the node is a target of a
-  // link, otherwise it'd be in a node_modules folder, so take advantage of
+  // link; otherwise, it'd be in a node_modules folder, so take advantage of
   // that to limit the scans later.
   #topNodes = new Set()
   #transplantFilter
 
-  constructor (options) {
-    super(options)
-
-    // the tree of nodes on disk
-    this.actualTree = options.actualTree
-
-    // caches for cached realpath calls
-    const cwd = process.cwd()
-    // assume that the cwd is real enough for our purposes
-    this[_rpcache] = new Map([[cwd, cwd]])
-    this[_stcache] = new Map()
-  }
-
   // public method
   // TODO remove options param in next semver major
   async loadActual (options = {}) {

package/lib/arborist/load-virtual.js

@@ -18,14 +18,6 @@ const setWorkspaces = Symbol.for('setWorkspaces')
 module.exports = cls => class VirtualLoader extends cls {
   #rootOptionProvided
 
-  constructor (options) {
-    super(options)
-
-    // the virtual tree we load from a shrinkwrap
-    this.virtualTree = options.virtualTree
-    this[flagsSuspect] = false
-  }
-
   // public method
   async loadVirtual (options = {}) {
     if (this.virtualTree) {
@@ -69,11 +61,7 @@ module.exports = cls => class VirtualLoader extends cls {
     if (!this.#rootOptionProvided) {
       // root is never any of these things, but might be a brand new
       // baby Node object that never had its dep flags calculated.
-      root.extraneous = false
-      root.dev = false
-      root.optional = false
-      root.devOptional = false
-      root.peer = false
+      root.unsetDepFlags()
     } else {
       this[flagsSuspect] = true
     }
@@ -81,7 +69,21 @@ module.exports = cls => class VirtualLoader extends cls {
     this.#checkRootEdges(s, root)
     root.meta = s
     this.virtualTree = root
-    const { links, nodes } = this.#resolveNodes(s, root)
+    // separate out link metadata, and create Node objects for nodes
+    const links = new Map()
+    const nodes = new Map([['', root]])
+    for (const [location, meta] of Object.entries(s.data.packages)) {
+      // skip the root because we already got it
+      if (!location) {
+        continue
+      }
+
+      if (meta.link) {
+        links.set(location, meta)
+      } else {
+        nodes.set(location, this.#loadNode(location, meta))
+      }
+    }
     await this.#resolveLinks(links, nodes)
     if (!(s.originalLockfileVersion >= 2)) {
       this.#assignBundles(nodes)
@@ -93,11 +95,7 @@ module.exports = cls => class VirtualLoader extends cls {
         if (node.isRoot || node === this.#rootOptionProvided) {
           continue
         }
-        node.extraneous = true
-        node.dev = true
-        node.optional = true
-        node.devOptional = true
-        node.peer = true
+        node.resetDepFlags()
       }
       calcDepFlags(this.virtualTree, !this.#rootOptionProvided)
     }
@@ -168,27 +166,9 @@ module.exports = cls => class VirtualLoader extends cls {
     }
   }
 
-  // separate out link metadatas, and create Node objects for nodes
-  #resolveNodes (s, root) {
-    const links = new Map()
-    const nodes = new Map([['', root]])
-    for (const [location, meta] of Object.entries(s.data.packages)) {
-      // skip the root because we already got it
-      if (!location) {
-        continue
-      }
-
-      if (meta.link) {
-        links.set(location, meta)
-      } else {
-        nodes.set(location, this.#loadNode(location, meta))
-      }
-    }
-    return { links, nodes }
-  }
-
   // links is the set of metadata, and nodes is the map of non-Link nodes
   // Set the targets to nodes in the set, if we have them (we might not)
+  // XXX build-ideal-tree also has a #resolveLinks, is there overlap?
   async #resolveLinks (links, nodes) {
     for (const [location, meta] of links.entries()) {
       const targetPath = resolve(this.path, meta.resolved)
@@ -255,11 +235,6 @@ To fix:
       sw.name = nameFromFolder(path)
     }
 
-    const dev = sw.dev
-    const optional = sw.optional
-    const devOptional = dev || optional || sw.devOptional
-    const peer = sw.peer
-
     const node = new Node({
       installLinks: this.installLinks,
       legacyPeerDeps: this.legacyPeerDeps,
@@ -269,20 +244,16 @@ To fix:
       integrity: sw.integrity,
       resolved: consistentResolve(sw.resolved, this.path, path),
       pkg: sw,
-      ideallyInert: sw.ideallyInert,
       hasShrinkwrap: sw.hasShrinkwrap,
-      dev,
-      optional,
-      devOptional,
-      peer,
       loadOverrides,
+      // cast to boolean because they're undefined in the lock file when false
+      extraneous: !!sw.extraneous,
+      devOptional: !!(sw.devOptional || sw.dev || sw.optional),
+      peer: !!sw.peer,
+      optional: !!sw.optional,
+      dev: !!sw.dev,
     })
-    // cast to boolean because they're undefined in the lock file when false
-    node.extraneous = !!sw.extraneous
-    node.devOptional = !!(sw.devOptional || sw.dev || sw.optional)
-    node.peer = !!sw.peer
-    node.optional = !!sw.optional
-    node.dev = !!sw.dev
+
     return node
   }
 

package/lib/arborist/rebuild.js

@@ -24,13 +24,12 @@ const _trashList = Symbol.for('trashList')
 module.exports = cls => class Builder extends cls {
   #doHandleOptionalFailure
   #oldMeta = null
-  #queues
-
-  constructor (options) {
-    super(options)
-
-    this.scriptsRun = new Set()
-    this.#resetQueues()
+  #queues = {
+    preinstall: [],
+    install: [],
+    postinstall: [],
+    prepare: [],
+    bin: [],
   }
 
   async rebuild ({ nodes, handleOptionalFailure = false } = {}) {
@@ -62,7 +61,13 @@ module.exports = cls => class Builder extends cls {
 
     // build link deps
     if (linkNodes.size) {
-      this.#resetQueues()
+      this.#queues = {
+        preinstall: [],
+        install: [],
+        postinstall: [],
+        prepare: [],
+        bin: [],
+      }
       await this.#build(linkNodes, { type: 'links' })
     }
 
@@ -132,16 +137,6 @@ module.exports = cls => class Builder extends cls {
     }
   }
 
-  #resetQueues () {
-    this.#queues = {
-      preinstall: [],
-      install: [],
-      postinstall: [],
-      prepare: [],
-      bin: [],
-    }
-  }
-
   async #build (nodes, { type = 'deps' }) {
     const timeEnd = time.start(`build:${type}`)