@npmcli/arborist 2.2.9 → 2.4.2

package/lib/diff.js

@@ -11,7 +11,9 @@ const {existsSync} = require('fs')
 const ssri = require('ssri')
 
 class Diff {
-  constructor ({actual, ideal}) {
+  constructor ({actual, ideal, filterSet, shrinkwrapInflated}) {
+    this.filterSet = filterSet
+    this.shrinkwrapInflated = shrinkwrapInflated
     this.children = []
     this.actual = actual
     this.ideal = ideal
@@ -29,9 +31,54 @@ class Diff {
     this.removed = []
   }
 
-  static calculate ({actual, ideal}) {
+  static calculate ({actual, ideal, filterNodes = [], shrinkwrapInflated = new Set()}) {
+    // if there's a filterNode, then:
+    // - get the path from the root to the filterNode.  The root or
+    //   root.target should have an edge either to the filterNode or
+    //   a link to the filterNode.  If not, abort.  Add the path to the
+    //   filterSet.
+    // - Add set of Nodes depended on by the filterNode to filterSet.
+    // - Anything outside of that set should be ignored by getChildren
+    const filterSet = new Set()
+    for (const filterNode of filterNodes) {
+      const { root } = filterNode
+      if (root !== ideal && root !== actual)
+        throw new Error('invalid filterNode: outside idealTree/actualTree')
+      const { target } = root
+      const rootTarget = target || root
+      const edge = [...rootTarget.edgesOut.values()].filter(e => {
+        return e.to && (e.to === filterNode || e.to.target === filterNode)
+      })[0]
+      filterSet.add(root)
+      filterSet.add(rootTarget)
+      filterSet.add(ideal)
+      filterSet.add(actual)
+      if (edge && edge.to) {
+        filterSet.add(edge.to)
+        if (edge.to.target)
+          filterSet.add(edge.to.target)
+      }
+      filterSet.add(filterNode)
+
+      depth({
+        tree: filterNode,
+        visit: node => filterSet.add(node),
+        getChildren: node => {
+          node = node.target || node
+          const loc = node.location
+          const idealNode = ideal.inventory.get(loc)
+          const ideals = !idealNode ? []
+            : [...idealNode.edgesOut.values()].filter(e => e.to).map(e => e.to)
+          const actualNode = actual.inventory.get(loc)
+          const actuals = !actualNode ? []
+            : [...actualNode.edgesOut.values()].filter(e => e.to).map(e => e.to)
+          return ideals.concat(actuals)
+        },
+      })
+    }
+
     return depth({
-      tree: new Diff({actual, ideal}),
+      tree: new Diff({actual, ideal, filterSet, shrinkwrapInflated}),
       getChildren,
       leave,
     })
@@ -89,20 +136,29 @@ const allChildren = node => {
 // to create the diff tree
 const getChildren = diff => {
   const children = []
-  const {unchanged, removed} = diff
+  const {actual, ideal, unchanged, removed, filterSet, shrinkwrapInflated} = diff
 
   // Note: we DON'T diff fsChildren themselves, because they are either
   // included in the package contents, or part of some other project, and
   // will never appear in legacy shrinkwraps anyway.  but we _do_ include the
   // child nodes of fsChildren, because those are nodes that we are typically
   // responsible for installing.
-  const actualKids = allChildren(diff.actual)
-  const idealKids = allChildren(diff.ideal)
+  const actualKids = allChildren(actual)
+  const idealKids = allChildren(ideal)
+
+  if (ideal && ideal.hasShrinkwrap && !shrinkwrapInflated.has(ideal)) {
+    // Guaranteed to get a diff.leaves here, because we always
+    // be called with a proper Diff object when ideal has a shrinkwrap
+    // that has not been inflated.
+    diff.leaves.push(diff)
+    return children
+  }
+
   const paths = new Set([...actualKids.keys(), ...idealKids.keys()])
   for (const path of paths) {
     const actual = actualKids.get(path)
     const ideal = idealKids.get(path)
-    diffNode(actual, ideal, children, unchanged, removed)
+    diffNode(actual, ideal, children, unchanged, removed, filterSet, shrinkwrapInflated)
   }
 
   if (diff.leaves && !children.length)
@@ -111,15 +167,18 @@ const getChildren = diff => {
   return children
 }
 
-const diffNode = (actual, ideal, children, unchanged, removed) => {
+const diffNode = (actual, ideal, children, unchanged, removed, filterSet, shrinkwrapInflated) => {
+  if (filterSet.size && !(filterSet.has(ideal) || filterSet.has(actual)))
+    return
+
   const action = getAction({actual, ideal})
 
   // if it's a match, then get its children
   // otherwise, this is the child diff node
-  if (action) {
+  if (action || (!shrinkwrapInflated.has(ideal) && ideal.hasShrinkwrap)) {
     if (action === 'REMOVE')
       removed.push(actual)
-    children.push(new Diff({actual, ideal}))
+    children.push(new Diff({actual, ideal, filterSet, shrinkwrapInflated}))
   } else {
     unchanged.push(ideal)
     // !*! Weird dirty hack warning !*!
@@ -150,7 +209,7 @@ const diffNode = (actual, ideal, children, unchanged, removed) => {
       for (const node of bundledChildren)
         node.parent = ideal
     }
-    children.push(...getChildren({actual, ideal, unchanged, removed}))
+    children.push(...getChildren({actual, ideal, unchanged, removed, filterSet, shrinkwrapInflated}))
   }
 }
 

package/lib/index.js

@@ -3,5 +3,6 @@ module.exports.Arborist = module.exports
 module.exports.Node = require('./node.js')
 module.exports.Link = require('./link.js')
 module.exports.Edge = require('./edge.js')
+module.exports.Shrinkwrap = require('./shrinkwrap.js')
 // XXX export the other classes, too.  shrinkwrap, diff, etc.
 // they're handy!

package/lib/inventory.js

@@ -4,7 +4,7 @@
 // keys is the set of fields to be able to query.
 const _primaryKey = Symbol('_primaryKey')
 const _index = Symbol('_index')
-const defaultKeys = ['name', 'license', 'funding', 'realpath']
+const defaultKeys = ['name', 'license', 'funding', 'realpath', 'packageName']
 const { hasOwnProperty } = Object.prototype
 const debug = require('./debug.js')
 class Inventory extends Map {

package/lib/link.js

@@ -23,13 +23,19 @@ class Link extends Node {
       : null),
     })
 
-    this.target = target || new Node({
-      ...options,
-      path: realpath,
-      parent: null,
-      fsParent: null,
-      root: this.root,
-    })
+    if (target)
+      this.target = target
+    else if (this.realpath === this.root.path)
+      this.target = this.root
+    else {
+      this.target = new Node({
+        ...options,
+        path: realpath,
+        parent: null,
+        fsParent: null,
+        root: this.root,
+      })
+    }
   }
 
   get version () {

package/lib/node.js

@@ -28,6 +28,7 @@
 // where we need to quickly find all instances of a given package name within a
 // tree.
 
+const semver = require('semver')
 const nameFromFolder = require('@npmcli/name-from-folder')
 const Edge = require('./edge.js')
 const Inventory = require('./inventory.js')
@@ -290,6 +291,10 @@ class Node {
     return this[_package].version || ''
   }
 
+  get packageName () {
+    return this[_package].name || null
+  }
+
   get pkgid () {
     const { name = '', version = '' } = this.package
     // root package will prefer package name over folder name,
@@ -349,10 +354,10 @@ class Node {
     }
 
     const why = {
-      name: this.isProjectRoot ? this.package.name : this.name,
+      name: this.isProjectRoot ? this.packageName : this.name,
       version: this.package.version,
     }
-    if (this.errors.length || !this.package.name || !this.package.version) {
+    if (this.errors.length || !this.packageName || !this.package.version) {
       why.errors = this.errors.length ? this.errors : [
         new Error('invalid package: lacks name and/or version'),
       ]
@@ -459,7 +464,7 @@ class Node {
     if (this.isProjectRoot)
       return false
     const { root } = this
-    const { type, to } = root.edgesOut.get(this.package.name) || {}
+    const { type, to } = root.edgesOut.get(this.packageName) || {}
     return type === 'workspace' && to && (to.target === this || to === this)
   }
 
@@ -685,6 +690,7 @@ class Node {
       ...this.children.values(),
       ...this.inventory.values(),
     ].filter(n => n !== this))
+
     for (const child of family) {
       if (child.root !== root) {
         child[_delistFromMeta]()
@@ -704,12 +710,14 @@ class Node {
     }
 
     // if we had a target, and didn't find one in the new root, then bring
-    // it over as well.
-    if (this.isLink && target && !this.target)
+    // it over as well, but only if we're setting the link into a new root,
+    // as we don't want to lose the target any time we remove a link.
+    if (this.isLink && target && !this.target && root !== this)
       target.root = root
 
     // tree should always be valid upon root setter completion.
     treeCheck(this)
+    treeCheck(root)
   }
 
   get root () {
@@ -726,20 +734,14 @@ class Node {
 
   [_loadDeps] () {
     // Caveat!  Order is relevant!
-    // packages in optionalDependencies and prod/peer/dev are
-    // optional.  Packages in both deps and devDeps are required.
+    // Packages in optionalDependencies are optional.
+    // Packages in both deps and devDeps are required.
     // Note the subtle breaking change from v6: it is no longer possible
     // to have a different spec for a devDep than production dep.
-    this[_loadDepType](this.package.optionalDependencies, 'optional')
 
     // Linked targets that are disconnected from the tree are tops,
     // but don't have a 'path' field, only a 'realpath', because we
     // don't know their canonical location. We don't need their devDeps.
-    const { isTop, path, sourceReference } = this
-    const { isTop: srcTop, path: srcPath } = sourceReference || {}
-    if (isTop && path && (!sourceReference || srcTop && srcPath))
-      this[_loadDepType](this.package.devDependencies, 'dev')
-
     const pd = this.package.peerDependencies
     if (pd && typeof pd === 'object' && !this.legacyPeerDeps) {
       const pm = this.package.peerDependenciesMeta || {}
@@ -756,19 +758,22 @@ class Node {
     }
 
     this[_loadDepType](this.package.dependencies, 'prod')
+    this[_loadDepType](this.package.optionalDependencies, 'optional')
+
+    const { isTop, path, sourceReference } = this
+    const { isTop: srcTop, path: srcPath } = sourceReference || {}
+    if (isTop && path && (!sourceReference || srcTop && srcPath))
+      this[_loadDepType](this.package.devDependencies, 'dev')
   }
 
-  [_loadDepType] (obj, type) {
-    const from = this
+  [_loadDepType] (deps, type) {
     const ad = this.package.acceptDependencies || {}
-    for (const [name, spec] of Object.entries(obj || {})) {
-      const accept = ad[name]
-      // if it's already set, then we keep the existing edge
-      // Prod deps should not be marked as dev, however.
-      // NB: the Edge ctor adds itself to from.edgesOut
+    // Because of the order in which _loadDeps runs, we always want to
+    // prioritize a new edge over an existing one
+    for (const [name, spec] of Object.entries(deps || {})) {
       const current = this.edgesOut.get(name)
-      if (!current || current.dev && type === 'prod')
-        new Edge({ from, name, spec, accept, type })
+      if (!current || current.type !== 'workspace')
+        new Edge({ from: this, name, spec, accept: ad[name], type })
     }
   }
 
@@ -882,6 +887,43 @@ class Node {
     return node.canReplaceWith(this)
   }
 
+  // return true if it's safe to remove this node, because anything that
+  // is depending on it would be fine with the thing that they would resolve
+  // to if it was removed, or nothing is depending on it in the first place.
+  canDedupe (preferDedupe = false) {
+    // not allowed to mess with shrinkwraps or bundles
+    if (this.inDepBundle || this.inShrinkwrap)
+      return false
+
+    // it's a top level pkg, or a dep of one
+    if (!this.parent || !this.parent.parent)
+      return false
+
+    // no one wants it, remove it
+    if (this.edgesIn.size === 0)
+      return true
+
+    const other = this.parent.parent.resolve(this.name)
+
+    // nothing else, need this one
+    if (!other)
+      return false
+
+    // if it's the same thing, then always fine to remove
+    if (other.matches(this))
+      return true
+
+    // if the other thing can't replace this, then skip it
+    if (!other.canReplace(this))
+      return false
+
+    // if we prefer dedupe, or if the version is greater/equal, take the other
+    if (preferDedupe || semver.gte(other.version, this.version))
+      return true
+
+    return false
+  }
+
   satisfies (requested) {
     if (requested instanceof Edge)
       return this.name === requested.name && requested.satisfiedBy(this)
@@ -924,8 +966,8 @@ class Node {
 
     // if no resolved, check both package name and version
     // otherwise, conclude that they are different things
-    return this.package.name && node.package.name &&
-      this.package.name === node.package.name &&
+    return this.packageName && node.packageName &&
+      this.packageName === node.packageName &&
       this.version && node.version &&
       this.version === node.version
   }

package/lib/printable.js

@@ -2,12 +2,13 @@
 // of the current node and its descendents
 
 const util = require('util')
+const relpath = require('./relpath.js')
 
 class ArboristNode {
   constructor (tree, path) {
     this.name = tree.name
-    if (tree.package.name && tree.package.name !== this.name)
-      this.packageName = tree.package.name
+    if (tree.packageName && tree.packageName !== this.name)
+      this.packageName = tree.packageName
     if (tree.version)
       this.version = tree.version
     this.location = tree.location
@@ -28,6 +29,15 @@ class ArboristNode {
       this.peer = true
     if (tree.inBundle)
       this.bundled = true
+    if (tree.inDepBundle)
+      this.bundler = tree.getBundler().location
+    const bd = tree.package && tree.package.bundleDependencies
+    if (bd && bd.length)
+      this.bundleDependencies = bd
+    if (tree.inShrinkwrap)
+      this.inShrinkwrap = true
+    else if (tree.hasShrinkwrap)
+      this.hasShrinkwrap = true
     if (tree.error)
       this.error = treeError(tree.error)
     if (tree.errors && tree.errors.length)
@@ -47,6 +57,11 @@ class ArboristNode {
         .map(edge => new EdgeIn(edge)))
     }
 
+    if (tree.workspaces && tree.workspaces.size) {
+      this.workspaces = new Map([...tree.workspaces.entries()]
+        .map(([name, path]) => [name, relpath(tree.root.realpath, path)]))
+    }
+
     // fsChildren sorted by path
     if (tree.fsChildren.size) {
       this.fsChildren = new Set([...tree.fsChildren]
@@ -126,6 +141,9 @@ class EdgeIn extends Edge {
 }
 
 const printableTree = (tree, path = []) => {
+  if (!tree)
+    return tree
+
   const Cls = tree.isLink ? ArboristLink
     : tree.sourceReference ? ArboristVirtualNode
     : ArboristNode

package/lib/shrinkwrap.js

@@ -254,7 +254,7 @@ class Shrinkwrap {
         meta[key.replace(/^_/, '')] = val
     })
     // we only include name if different from the node path name
-    const pname = node.package.name
+    const pname = node.packageName
     if (pname && pname !== node.name)
       meta.name = pname
 
@@ -825,7 +825,7 @@ class Shrinkwrap {
   [_buildLegacyLockfile] (node, lock, path = []) {
     if (node === this.tree) {
       // the root node
-      lock.name = node.package.name || node.name
+      lock.name = node.packageName || node.name
       if (node.version)
         lock.version = node.version
     }
@@ -870,9 +870,9 @@ class Shrinkwrap {
         lock.from = spec.raw
     } else if (!node.isRoot &&
         node.package &&
-        node.package.name &&
-        node.package.name !== node.name)
-      lock.version = `npm:${node.package.name}@${node.version}`
+        node.packageName &&
+        node.packageName !== node.name)
+      lock.version = `npm:${node.packageName}@${node.version}`
     else if (node.package && node.version)
       lock.version = node.version
 

package/lib/tree-check.js

@@ -1,6 +1,8 @@
 const debug = require('./debug.js')
 
 const checkTree = (tree, checkUnreachable = true) => {
+  const log = [['START TREE CHECK', tree.path]]
+
   // this can only happen in tests where we have a "tree" object
   // that isn't actually a tree.
   if (!tree.root || !tree.root.inventory)
@@ -9,8 +11,21 @@ const checkTree = (tree, checkUnreachable = true) => {
   const { inventory } = tree.root
   const seen = new Set()
   const check = (node, via = tree, viaType = 'self') => {
+    log.push([
+      'CHECK',
+      node && node.location,
+      via && via.location,
+      viaType,
+      'seen=' + seen.has(node),
+      'promise=' + !!(node && node.then),
+      'root=' + !!(node && node.isRoot),
+    ])
+
     if (!node || seen.has(node) || node.then)
       return
+
+    seen.add(node)
+
     if (node.isRoot && node !== tree.root) {
       throw Object.assign(new Error('double root'), {
         node: node.path,
@@ -19,6 +34,7 @@ const checkTree = (tree, checkUnreachable = true) => {
         root: tree.root.path,
         via: via.path,
         viaType,
+        log,
       })
     }
 
@@ -31,6 +47,7 @@ const checkTree = (tree, checkUnreachable = true) => {
         via: via.path,
         viaType,
         otherRoot: node.root && node.root.path,
+        log,
       })
     }
 
@@ -43,6 +60,7 @@ const checkTree = (tree, checkUnreachable = true) => {
         viaType,
         inventory: [...node.inventory.values()].map(node =>
           [node.path, node.location]),
+        log,
       })
     }
 
@@ -53,6 +71,7 @@ const checkTree = (tree, checkUnreachable = true) => {
         root: tree.root.path,
         via: via.path,
         viaType,
+        log,
       })
     }
 
@@ -65,14 +84,38 @@ const checkTree = (tree, checkUnreachable = true) => {
         via: via.path,
         viaType,
         devEdges: devEdges.map(e => [e.type, e.name, e.spec, e.error]),
+        log,
+      })
+    }
+
+    if (node.path === tree.root.path && node !== tree.root) {
+      throw Object.assign(new Error('node with same path as root'), {
+        node: node.path,
+        tree: tree.path,
+        root: tree.root.path,
+        via: via.path,
+        viaType,
+        log,
+      })
+    }
+
+    if (!node.isLink && node.path !== node.realpath) {
+      throw Object.assign(new Error('non-link with mismatched path/realpath'), {
+        node: node.path,
+        tree: tree.path,
+        realpath: node.realpath,
+        root: tree.root.path,
+        via: via.path,
+        viaType,
+        log,
       })
     }
 
     const { parent, fsParent, target } = node
-    seen.add(node)
     check(parent, node, 'parent')
     check(fsParent, node, 'fsParent')
     check(target, node, 'target')
+    log.push(['CHILDREN', node.location, ...node.children.keys()])
     for (const kid of node.children.values())
       check(kid, node, 'children')
     for (const kid of node.fsChildren)
@@ -81,6 +124,7 @@ const checkTree = (tree, checkUnreachable = true) => {
       check(link, node, 'linksIn')
     for (const top of node.tops)
       check(top, node, 'tops')
+    log.push(['DONE', node.location])
   }
   check(tree)
   if (checkUnreachable) {
@@ -92,6 +136,7 @@ const checkTree = (tree, checkUnreachable = true) => {
           location: node.location,
           root: tree.root.path,
           tree: tree.path,
+          log,
         })
       }
     }

package/lib/update-root-package-json.js

@@ -6,8 +6,6 @@ const {resolve} = require('path')
 
 const parseJSON = require('json-parse-even-better-errors')
 
-const { orderDeps } = require('./dep-spec.js')
-
 const depTypes = new Set([
   'dependencies',
   'optionalDependencies',
@@ -15,6 +13,20 @@ const depTypes = new Set([
   'peerDependencies',
 ])
 
+// sort alphabetically all types of deps for a given package
+const orderDeps = (pkg) => {
+  for (const type of depTypes) {
+    if (pkg && pkg[type]) {
+      pkg[type] = Object.keys(pkg[type])
+        .sort((a, b) => a.localeCompare(b))
+        .reduce((res, key) => {
+          res[key] = pkg[type][key]
+          return res
+        }, {})
+    }
+  }
+  return pkg
+}
 const parseJsonSafe = json => {
   try {
     return parseJSON(json)

package/lib/vuln.js

@@ -83,6 +83,9 @@ class Vuln {
     if (!specObj.registry)
       return true
 
+    if (specObj.subSpec)
+      spec = specObj.subSpec.rawSpec
+
     for (const v of this.versions) {
       if (satisfies(v, spec) && !satisfies(v, this.range, semverOpt))
         return false

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@npmcli/arborist",
-  "version": "2.2.9",
+  "version": "2.4.2",
   "description": "Manage node_modules trees",
   "dependencies": {
     "@npmcli/installed-package-contents": "^1.0.7",
@@ -14,12 +14,12 @@
     "cacache": "^15.0.3",
     "common-ancestor-path": "^1.0.1",
     "json-parse-even-better-errors": "^2.3.1",
-    "json-stringify-nice": "^1.1.1",
+    "json-stringify-nice": "^1.1.2",
     "mkdirp-infer-owner": "^2.0.0",
     "npm-install-checks": "^4.0.0",
     "npm-package-arg": "^8.1.0",
     "npm-pick-manifest": "^6.1.0",
-    "npm-registry-fetch": "^9.0.0",
+    "npm-registry-fetch": "^10.0.0",
     "pacote": "^11.2.6",
     "parse-conflict-json": "^1.1.1",
     "promise-all-reject-late": "^1.0.0",
@@ -41,8 +41,7 @@
     "eslint-plugin-standard": "^4.0.1",
     "minify-registry-metadata": "^2.1.0",
     "mutate-fs": "^2.1.1",
-    "require-inject": "^1.4.4",
-    "tap": "^14.11.0",
+    "tap": "^15.0.4",
     "tcompare": "^3.0.4"
   },
   "scripts": {
@@ -76,10 +75,8 @@
     "arborist": "bin/index.js"
   },
   "tap": {
-    "100": true,
     "after": "test/fixtures/cleanup.js",
     "coverage-map": "map.js",
-    "esm": false,
     "test-env": [
       "NODE_OPTIONS=--no-warnings"
     ],
@@ -87,6 +84,6 @@
       "--no-warnings",
       "--no-deprecation"
     ],
-    "timeout": "120"
+    "timeout": "240"
   }
 }

package/lib/dep-spec.js

@@ -1,43 +0,0 @@
-const types = [
-  'peerDependencies',
-  'devDependencies',
-  'optionalDependencies',
-  'dependencies',
-]
-
-const findType = (pkg, name) => {
-  for (const t of types) {
-    if (pkg[t] && typeof pkg[t] === 'object' && pkg[t][name] !== undefined)
-      return t
-  }
-  return 'dependencies'
-}
-
-// given a dep name and spec, update it wherever it exists in
-// the manifest, or add the spec to 'dependencies' if not found.
-const updateDepSpec = (pkg, name, newSpec) => {
-  const type = findType(pkg, name)
-  pkg[type] = pkg[type] || {}
-  pkg[type][name] = newSpec
-  return pkg
-}
-
-// sort alphabetically all types of deps for a given package
-const orderDeps = (pkg) => {
-  for (const type of types) {
-    if (pkg && pkg[type]) {
-      pkg[type] = Object.keys(pkg[type])
-        .sort((a, b) => a.localeCompare(b))
-        .reduce((res, key) => {
-          res[key] = pkg[type][key]
-          return res
-        }, {})
-    }
-  }
-  return pkg
-}
-
-module.exports = {
-  orderDeps,
-  updateDepSpec,
-}