@npmcli/arborist 2.4.0 → 2.4.4

package/bin/license.js

@@ -22,7 +22,7 @@ a.loadVirtual().then(tree => {
       set.push([tree.inventory.query('license', license).size, license])
 
     for (const [count, license] of set.sort((a, b) =>
-      a[1] && b[1] ? b[0] - a[0] || a[1].localeCompare(b[1])
+      a[1] && b[1] ? b[0] - a[0] || a[1].localeCompare(b[1], 'en')
       : a[1] ? -1
       : b[1] ? 1
       : 0))

package/lib/add-rm-pkg-deps.js

@@ -1,60 +1,60 @@
 // add and remove dependency specs to/from pkg manifest
 
-const removeFromOthers = (name, type, pkg) => {
-  const others = new Set([
-    'dependencies',
-    'optionalDependencies',
-    'devDependencies',
-    'peerDependenciesMeta',
-    'peerDependencies',
-  ])
-
-  switch (type) {
-    case 'prod':
-      others.delete('dependencies')
-      break
-    case 'dev':
-      others.delete('devDependencies')
-      others.delete('peerDependencies')
-      others.delete('peerDependenciesMeta')
-      break
-    case 'optional':
-      others.delete('optionalDependencies')
-      break
-    case 'peer':
-    case 'peerOptional':
-      others.delete('devDependencies')
-      others.delete('peerDependencies')
-      others.delete('peerDependenciesMeta')
-      break
-  }
-
-  for (const other of others)
-    deleteSubKey(pkg, other, name)
-}
-
-const add = ({pkg, add, saveBundle, saveType}) => {
+const add = ({pkg, add, saveBundle, saveType, log}) => {
   for (const spec of add)
-    addSingle({pkg, spec, saveBundle, saveType})
+    addSingle({pkg, spec, saveBundle, saveType, log})
 
   return pkg
 }
 
-const addSingle = ({pkg, spec, saveBundle, saveType}) => {
-  if (!saveType)
-    saveType = getSaveType(pkg, spec)
+// Canonical source of both the map between saveType and where it correlates to
+// in the package, and the names of all our dependencies attributes
+const saveTypeMap = new Map([
+  ['dev', 'devDependencies'],
+  ['optional', 'optionalDependencies'],
+  ['prod', 'dependencies'],
+  ['peerOptional', 'peerDependencies'],
+  ['peer', 'peerDependencies'],
+])
 
+const addSingle = ({pkg, spec, saveBundle, saveType, log}) => {
   const { name, rawSpec } = spec
-  removeFromOthers(name, saveType, pkg)
-  const type = saveType === 'prod' ? 'dependencies'
-    : saveType === 'optional' ? 'optionalDependencies'
-    : saveType === 'peer' || saveType === 'peerOptional' ? 'peerDependencies'
-    : saveType === 'dev' ? 'devDependencies'
-    : /* istanbul ignore next */ null
 
-  pkg[type] = pkg[type] || {}
-  if (rawSpec !== '' || pkg[type][name] === undefined)
-    pkg[type][name] = rawSpec || '*'
+  // if the user does not give us a type, we infer which type(s)
+  // to keep based on the same order of priority we do when
+  // building the tree as defined in the _loadDeps method of
+  // the node class.
+  if (!saveType)
+    saveType = inferSaveType(pkg, spec.name)
+
+  if (saveType === 'prod') {
+    // a production dependency can only exist as production (rpj ensures it
+    // doesn't coexist w/ optional)
+    deleteSubKey(pkg, 'devDependencies', name, 'dependencies', log)
+    deleteSubKey(pkg, 'peerDependencies', name, 'dependencies', log)
+  } else if (saveType === 'dev') {
+    // a dev dependency may co-exist as peer, or optional, but not production
+    deleteSubKey(pkg, 'dependencies', name, 'devDependencies', log)
+  } else if (saveType === 'optional') {
+    // an optional dependency may co-exist as dev (rpj ensures it doesn't
+    // coexist w/ prod)
+    deleteSubKey(pkg, 'peerDependencies', name, 'optionalDependencies', log)
+  } else { // peer or peerOptional is all that's left
+    // a peer dependency may coexist as dev
+    deleteSubKey(pkg, 'dependencies', name, 'peerDependencies', log)
+    deleteSubKey(pkg, 'optionalDependencies', name, 'peerDependencies', log)
+  }
+
+  const depType = saveTypeMap.get(saveType)
+
+  pkg[depType] = pkg[depType] || {}
+  if (rawSpec !== '' || pkg[depType][name] === undefined)
+    pkg[depType][name] = rawSpec || '*'
+  if (saveType === 'optional') {
+    // Affordance for previous npm versions that require this behaviour
+    pkg.dependencies = pkg.dependencies || {}
+    pkg.dependencies[name] = pkg.optionalDependencies[name]
+  }
 
   if (saveType === 'peer' || saveType === 'peerOptional') {
     const pdm = pkg.peerDependenciesMeta || {}
@@ -75,51 +75,53 @@ const addSingle = ({pkg, spec, saveBundle, saveType}) => {
     // keep it sorted, keep it unique
     const bd = new Set(pkg.bundleDependencies || [])
     bd.add(spec.name)
-    pkg.bundleDependencies = [...bd].sort((a, b) => a.localeCompare(b))
+    pkg.bundleDependencies = [...bd].sort((a, b) => a.localeCompare(b, 'en'))
   }
 }
 
-const getSaveType = (pkg, spec) => {
-  const {name} = spec
-  const {
-    // these names are so lonnnnngggg
-    devDependencies: devDeps,
-    optionalDependencies: optDeps,
-    peerDependencies: peerDeps,
-    peerDependenciesMeta: peerDepsMeta,
-  } = pkg
-
-  if (peerDeps && peerDeps[name] !== undefined) {
-    if (peerDepsMeta && peerDepsMeta[name] && peerDepsMeta[name].optional)
-      return 'peerOptional'
-    else
-      return 'peer'
-  } else if (devDeps && devDeps[name] !== undefined)
-    return 'dev'
-  else if (optDeps && optDeps[name] !== undefined)
-    return 'optional'
-  else
-    return 'prod'
+// Finds where the package is already in the spec and infers saveType from that
+const inferSaveType = (pkg, name) => {
+  for (const saveType of saveTypeMap.keys()) {
+    if (hasSubKey(pkg, saveTypeMap.get(saveType), name)) {
+      if (
+        saveType === 'peerOptional' &&
+        (!hasSubKey(pkg, 'peerDependenciesMeta', name) ||
+        !pkg.peerDependenciesMeta[name].optional)
+      )
+        return 'peer'
+      return saveType
+    }
+  }
+  return 'prod'
 }
 
-const deleteSubKey = (obj, k, sk) => {
-  if (obj[k]) {
-    delete obj[k][sk]
-    if (!Object.keys(obj[k]).length)
-      delete obj[k]
+const hasSubKey = (pkg, depType, name) => {
+  return pkg[depType] && Object.prototype.hasOwnProperty.call(pkg[depType], name)
+}
+
+// Removes a subkey and warns about it if it's being replaced
+const deleteSubKey = (pkg, depType, name, replacedBy, log) => {
+  if (hasSubKey(pkg, depType, name)) {
+    if (replacedBy && log)
+      log.warn('idealTree', `Removing ${depType}.${name} in favor of ${replacedBy}.${name}`)
+    delete pkg[depType][name]
+
+    // clean up peerDependenciesMeta if we are removing something from peerDependencies
+    if (depType === 'peerDependencies' && pkg.peerDependenciesMeta) {
+      delete pkg.peerDependenciesMeta[name]
+      if (!Object.keys(pkg.peerDependenciesMeta).length)
+        delete pkg.peerDependenciesMeta
+    }
+
+    if (!Object.keys(pkg[depType]).length)
+      delete pkg[depType]
   }
 }
 
 const rm = (pkg, rm) => {
-  for (const type of [
-    'dependencies',
-    'optionalDependencies',
-    'peerDependencies',
-    'peerDependenciesMeta',
-    'devDependencies',
-  ]) {
+  for (const depType of new Set(saveTypeMap.values())) {
     for (const name of rm)
-      deleteSubKey(pkg, type, name)
+      deleteSubKey(pkg, depType, name)
   }
   if (pkg.bundleDependencies) {
     pkg.bundleDependencies = pkg.bundleDependencies
@@ -130,4 +132,4 @@ const rm = (pkg, rm) => {
   return pkg
 }
 
-module.exports = { add, rm }
+module.exports = { add, rm, saveTypeMap, hasSubKey }

package/lib/arborist/build-ideal-tree.js

@@ -47,6 +47,7 @@ const _flagsSuspect = Symbol.for('flagsSuspect')
 const _workspaces = Symbol.for('workspaces')
 const _prune = Symbol('prune')
 const _preferDedupe = Symbol('preferDedupe')
+const _pruneDedupable = Symbol('pruneDedupable')
 const _legacyBundling = Symbol('legacyBundling')
 const _parseSettings = Symbol('parseSettings')
 const _initTree = Symbol('initTree')
@@ -503,6 +504,7 @@ module.exports = cls => class IdealTreeBuilder extends cls {
         saveBundle,
         saveType,
         path: this.path,
+        log: this.log,
       })
     })
   }
@@ -762,7 +764,7 @@ This is a one-time fix-up, please be patient...
     // sort physically shallower deps up to the front of the queue,
     // because they'll affect things deeper in, then alphabetical
     this[_depsQueue].sort((a, b) =>
-      (a.depth - b.depth) || a.path.localeCompare(b.path))
+      (a.depth - b.depth) || a.path.localeCompare(b.path, 'en'))
 
     const node = this[_depsQueue].shift()
     const bd = node.package.bundleDependencies
@@ -900,7 +902,7 @@ This is a one-time fix-up, please be patient...
     }
 
     const placed = tasks
-      .sort((a, b) => a.edge.name.localeCompare(b.edge.name))
+      .sort((a, b) => a.edge.name.localeCompare(b.edge.name, 'en'))
       .map(({ edge, dep }) => this[_placeDep](dep, node, edge))
 
     const promises = []
@@ -1145,7 +1147,7 @@ This is a one-time fix-up, please be patient...
       // we typically only install non-optional peers, but we have to
       // factor them into the peerSet so that we can avoid conflicts
       .filter(e => e.peer && !(e.valid && e.to))
-      .sort(({name: a}, {name: b}) => a.localeCompare(b))
+      .sort(({name: a}, {name: b}) => a.localeCompare(b, 'en'))
 
     for (const edge of peerEdges) {
       // already placed this one, and we're happy with it.
@@ -1342,6 +1344,21 @@ This is a one-time fix-up, please be patient...
         // this is an overridden peer dep
         this[_warnPeerConflict](edge)
       }
+
+      // if we get a KEEP in a update scenario, then we MAY have something
+      // already duplicating this unnecessarily!  For example:
+      // ```
+      // root
+      // +-- x (dep: y@1.x)
+      // |   +-- y@1.0.0
+      // +-- y@1.1.0
+      // ```
+      // Now say we do `reify({update:['y']})`, and the latest version is
+      // 1.1.0, which we already have in the root.  We'll try to place y@1.1.0
+      // first in x, then in the root, ending with KEEP, because we already
+      // have it.  In that case, we ought to REMOVE the nm/x/nm/y node, because
+      // it is an unnecessary duplicate.
+      this[_pruneDedupable](target, true)
       return []
     }
 
@@ -1397,8 +1414,8 @@ This is a one-time fix-up, please be patient...
     // MAY end up putting a better/identical node further up the tree in
     // a way that causes an unnecessary duplication.  If so, remove the
     // now-unnecessary node.
-    if (edge.valid && edge.to.parent !== target && newDep.canReplace(edge.to))
-      edge.to.parent = null
+    if (edge.valid && edge.to && edge.to !== newDep)
+      this[_pruneDedupable](edge.to, false)
 
     // visit any dependents who are upset by this change
     // if it's an angry overridden peer edge, however, make sure we
@@ -1414,30 +1431,8 @@ This is a one-time fix-up, please be patient...
     // prune anything deeper in the tree that can be replaced by this
     if (this.idealTree) {
       for (const node of this.idealTree.inventory.query('name', newDep.name)) {
-        if (node !== newDep &&
-            node.isDescendantOf(target) &&
-            !node.inShrinkwrap &&
-            !node.inBundle &&
-            node.canReplaceWith(newDep)) {
-          // don't prune if the dupe is necessary!
-          // root (a, d)
-          // +-- a (b, c2)
-          // |   +-- b (c2) <-- place c2 for b, lands at root
-          // +-- d (e)
-          //     +-- e (c1, d)
-          //         +-- c1
-          //         +-- f (c2)
-          //             +-- c2 <-- pruning this would be bad
-
-          const mask = node.parent !== target &&
-            node.parent &&
-            node.parent.parent &&
-            node.parent.parent !== target &&
-            node.parent.parent.resolve(newDep.name)
-
-          if (!mask || mask === newDep || node.canReplaceWith(mask))
-            node.parent = null
-        }
+        if (node.isDescendantOf(target))
+          this[_pruneDedupable](node, false)
       }
     }
 
@@ -1470,6 +1465,21 @@ This is a one-time fix-up, please be patient...
     return placed
   }
 
+  // prune all the nodes in a branch of the tree that can be safely removed
+  // This is only the most basic duplication detection; it finds if there
+  // is another satisfying node further up the tree, and if so, dedupes.
+  // Even in legacyBundling mode, we do this amount of deduplication.
+  [_pruneDedupable] (node, descend = true) {
+    if (node.canDedupe(this[_preferDedupe])) {
+      node.root = null
+      return
+    }
+    if (descend) {
+      for (const child of node.children.values())
+        this[_pruneDedupable](child)
+    }
+  }
+
   [_pruneForReplacement] (node, oldDeps) {
     // gather up all the invalid edgesOut, and any now-extraneous
     // deps that the new node doesn't depend on but the old one did.
@@ -1622,32 +1632,137 @@ This is a one-time fix-up, please be patient...
         // placed here as well.  the virtualRoot already has the appropriate
         // overrides applied.
         if (peerEntryEdge) {
-          const peerSet = getPeerSet(current)
-          OUTER: for (const p of peerSet) {
-            // if any have a non-peer dep from the target, or a peer dep if
-            // the target is root, then cannot safely replace and dupe deeper.
+          const currentPeerSet = getPeerSet(current)
+
+          // We are effectively replacing currentPeerSet with newPeerSet
+          // If there are any non-peer deps coming into the currentPeerSet,
+          // which are currently valid, and are from the target, then that
+          // means that we have to ensure that they're not going to be made
+          // invalid by putting the newPeerSet in place.
+          // If the edge comes from somewhere deeper than the target, then
+          // that's fine, because we'll create an invalid edge, detect it,
+          // and duplicate the node further into the tree.
+          // loop through the currentPeerSet checking for valid edges on
+          // the members of the peer set which will be made invalid.
+          const targetEdges = new Set()
+          for (const p of currentPeerSet) {
             for (const edge of p.edgesIn) {
-              if (peerSet.has(edge.from))
+              // edge from within the peerSet, ignore
+              if (currentPeerSet.has(edge.from))
+                continue
+              // only care about valid edges from target.
+              // edges from elsewhere can dupe if offended, invalid edges
+              // are already being fixed or will be later.
+              if (edge.from !== target || !edge.valid)
                 continue
+              targetEdges.add(edge)
+            }
+          }
 
-              // only respect valid edges, however, since we're likely trying
-              // to fix the very one that's currently broken!  If the virtual
-              // root's replacement is ok, and doesn't have any invalid edges
-              // indicating that it was an overridden peer, then ignore the
-              // conflict and continue.  If it WAS an override, then we need
-              // to get the conflict here so that we can decide whether to
-              // accept the current dep node, clobber it, or fail the install.
-              if (edge.from === target && edge.valid) {
-                const rep = dep.parent.children.get(edge.name)
-                const override = rep && ([...rep.edgesIn].some(e => !e.valid))
-                if (!rep || !rep.satisfies(edge) || override) {
+          for (const edge of targetEdges) {
+            // see if we intend to replace this one anyway
+            const rep = dep.parent.children.get(edge.name)
+            const current = edge.to
+            if (!rep) {
+              // this isn't one we're replacing.  but it WAS included in the
+              // peerSet for some reason, so make sure that it's still
+              // ok with the replacements in the new peerSet
+              for (const curEdge of current.edgesOut.values()) {
+                const newRepDep = dep.parent.children.get(curEdge.name)
+                if (curEdge.valid && newRepDep && !newRepDep.satisfies(curEdge)) {
                   canReplace = false
-                  break OUTER
+                  break
+                }
+              }
+              continue
+            }
+
+            // was this replacement already an override of some sort?
+            const override = [...rep.edgesIn].some(e => !e.valid)
+            // if we have a rep, and it's ok to put in this location, and
+            // it's not already part of an override in the peerSet, then
+            // we can continue with it.
+            if (rep.satisfies(edge) && !override)
+              continue
+            // Otherwise, we cannot replace.
+            canReplace = false
+            break
+          }
+          // if we're going to be replacing the peerSet, we have to remove
+          // and re-resolve any members of the old peerSet that are not
+          // present in the new one, and which will have invalid edges.
+          // We know that they're not depended upon by the target, or else
+          // they would have caused a conflict, so they'll get landed deeper
+          // in the tree, if possible.
+          if (canReplace) {
+            let needNesting = false
+            OUTER: for (const node of currentPeerSet) {
+              const rep = dep.parent.children.get(node.name)
+              // has a replacement, already addressed above
+              if (rep)
+                continue
+
+              // ok, it has been placed here to dedupe, see if it needs to go
+              // back deeper within the tree.
+              for (const edge of node.edgesOut.values()) {
+                const repDep = dep.parent.children.get(edge.name)
+                // not in new peerSet, maybe fine.
+                if (!repDep)
+                  continue
+
+                // new thing will be fine, no worries
+                if (repDep.satisfies(edge))
+                  continue
+
+                // uhoh, we'll have to nest them.
+                needNesting = true
+                break OUTER
+              }
+            }
+
+            // to nest, just delete everything without a target dep
+            // that's in the current peerSet, and add their dependants
+            // to the _depsQueue for evaluation.  Some of these MAY end
+            // up in the same location again, and that's fine.
+            if (needNesting) {
+              // avoid mutating the tree while we're examining it
+              const dependants = new Set()
+              const reresolve = new Set()
+              OUTER: for (const node of currentPeerSet) {
+                const rep = dep.parent.children.get(node.name)
+                if (rep)
+                  continue
+                // create a separate set for each one, so we can skip any
+                // that might somehow have an incoming target edge
+                const deps = new Set()
+                for (const edge of node.edgesIn) {
+                  // a target dep, skip this dep entirely, already addressed
+                  // ignoring for coverage, because it really ought to be
+                  // impossible, but I can't prove it yet, so this is here
+                  // for safety.
+                  /* istanbul ignore if - should be impossible */
+                  if (edge.from === target)
+                    continue OUTER
+                  // ignore this edge, it'll either be replaced or re-resolved
+                  if (currentPeerSet.has(edge.from))
+                    continue
+                  // ok, we care about this one.
+                  deps.add(edge.from)
                 }
+                reresolve.add(node)
+                for (const d of deps)
+                  dependants.add(d)
               }
+              for (const dependant of dependants) {
+                this[_depsQueue].push(dependant)
+                this[_depsSeen].delete(dependant)
+              }
+              for (const node of reresolve)
+                node.root = null
             }
           }
         }
+
         if (canReplace) {
           const ret = this[_canPlacePeers](dep, target, edge, REPLACE, peerEntryEdge, peerPath, isSource)
           /* istanbul ignore else - extremely rare that the peer set would

package/lib/arborist/index.js

@@ -29,6 +29,7 @@
 const {resolve} = require('path')
 const {homedir} = require('os')
 const procLog = require('../proc-log.js')
+const { saveTypeMap } = require('../add-rm-pkg-deps.js')
 
 const mixins = [
   require('../tracker.js'),
@@ -57,6 +58,8 @@ class Arborist extends Base {
       packumentCache: options.packumentCache || new Map(),
       log: options.log || procLog,
     }
+    if (options.saveType && !saveTypeMap.get(options.saveType))
+      throw new Error(`Invalid saveType ${options.saveType}`)
     this.cache = resolve(this.options.cache)
     this.path = resolve(this.options.path)
     process.emit('timeEnd', 'arborist:ctor')

package/lib/arborist/load-virtual.js

@@ -159,12 +159,12 @@ module.exports = cls => class VirtualLoader extends cls {
       ...depsToEdges('peerOptional', peerOptional),
       ...lockWS,
     ].sort(([atype, aname], [btype, bname]) =>
-      atype.localeCompare(btype) || aname.localeCompare(bname))
+      atype.localeCompare(btype, 'en') || aname.localeCompare(bname, 'en'))
 
     const rootEdges = [...root.edgesOut.values()]
       .map(e => [e.type, e.name, e.spec])
       .sort(([atype, aname], [btype, bname]) =>
-        atype.localeCompare(btype) || aname.localeCompare(bname))
+        atype.localeCompare(btype, 'en') || aname.localeCompare(bname, 'en'))
 
     if (rootEdges.length !== lockEdges.length) {
       // something added or removed

package/lib/arborist/rebuild.js

@@ -14,7 +14,7 @@ const {
 } = require('@npmcli/node-gyp')
 
 const boolEnv = b => b ? '1' : ''
-const sortNodes = (a, b) => (a.depth - b.depth) || a.path.localeCompare(b.path)
+const sortNodes = (a, b) => (a.depth - b.depth) || a.path.localeCompare(b.path, 'en')
 
 const _build = Symbol('build')
 const _resetQueues = Symbol('resetQueues')

package/lib/arborist/reify.js

@@ -3,9 +3,8 @@
 const onExit = require('../signal-handling.js')
 const pacote = require('pacote')
 const rpj = require('read-package-json-fast')
-const { updateDepSpec } = require('../dep-spec.js')
 const AuditReport = require('../audit-report.js')
-const {subset} = require('semver')
+const {subset, intersects} = require('semver')
 const npa = require('npm-package-arg')
 
 const {dirname, resolve, relative} = require('path')
@@ -28,6 +27,7 @@ const promiseAllRejectLate = require('promise-all-reject-late')
 const optionalSet = require('../optional-set.js')
 const updateRootPackageJson = require('../update-root-package-json.js')
 const calcDepFlags = require('../calc-dep-flags.js')
+const { saveTypeMap, hasSubKey } = require('../add-rm-pkg-deps.js')
 
 const _retiredPaths = Symbol('retiredPaths')
 const _retiredUnchanged = Symbol('retiredUnchanged')
@@ -406,11 +406,14 @@ module.exports = cls => class Reifier extends cls {
       return
 
     process.emit('time', 'reify:trashOmits')
+    // node.parent is checked to make sure this is a node that's in the tree, and
+    // not the parent-less top level nodes
     const filter = node =>
-      node.peer && this[_omitPeer] ||
-      node.dev && this[_omitDev] ||
-      node.optional && this[_omitOptional] ||
-      node.devOptional && this[_omitOptional] && this[_omitDev]
+      node.isDescendantOf(this.idealTree) &&
+        (node.peer && this[_omitPeer] ||
+          node.dev && this[_omitDev] ||
+          node.optional && this[_omitOptional] ||
+          node.devOptional && this[_omitOptional] && this[_omitDev])
 
     for (const node of this.idealTree.inventory.filter(filter))
       this[_addNodeToTrashList](node)
@@ -539,8 +542,8 @@ module.exports = cls => class Reifier extends cls {
     // Do the best with what we have, or else remove it from the tree
     // entirely, since we can't possibly reify it.
     const res = node.resolved ? `${node.name}@${this[_registryResolved](node.resolved)}`
-      : node.package.name && node.version
-        ? `${node.package.name}@${node.version}`
+      : node.packageName && node.version
+        ? `${node.packageName}@${node.version}`
         : null
 
     // no idea what this thing is.  remove it from the tree.
@@ -959,6 +962,7 @@ module.exports = cls => class Reifier extends cls {
         const spec = subSpec ? subSpec.rawSpec : rawSpec
         const child = root.children.get(name)
 
+        let newSpec
         if (req.registry) {
           const version = child.version
           const prefixRange = version ? this[_savePrefix] + version : '*'
@@ -968,18 +972,26 @@ module.exports = cls => class Reifier extends cls {
           // would allow versions outside the requested range.  Tags and
           // specific versions save with the save-prefix.
           const isRange = (subSpec || req).type === 'range'
-          const range = !isRange || subset(prefixRange, spec, { loose: true })
-            ? prefixRange : spec
-          const pname = child.package.name
+
+          let range = spec
+          if (
+            !isRange ||
+            spec === '*' ||
+            subset(prefixRange, spec, { loose: true })
+          )
+            range = prefixRange
+
+          const pname = child.packageName
           const alias = name !== pname
-          updateDepSpec(pkg, name, (alias ? `npm:${pname}@` : '') + range)
+          newSpec = alias ? `npm:${pname}@${range}` : range
         } else if (req.hosted) {
           // save the git+https url if it has auth, otherwise shortcut
           const h = req.hosted
           const opt = { noCommittish: false }
-          const save = h.https && h.auth ? `git+${h.https(opt)}`
-            : h.shortcut(opt)
-          updateDepSpec(pkg, name, save)
+          if (h.https && h.auth)
+            newSpec = `git+${h.https(opt)}`
+          else
+            newSpec = h.shortcut(opt)
         } else if (req.type === 'directory' || req.type === 'file') {
           // save the relative path in package.json
           // Normally saveSpec is updated with the proper relative
@@ -988,9 +1000,37 @@ module.exports = cls => class Reifier extends cls {
           // thing, so just get the ultimate fetchSpec and relativize it.
           const p = req.fetchSpec.replace(/^file:/, '')
           const rel = relpath(root.realpath, p)
-          updateDepSpec(pkg, name, `file:${rel}`)
+          newSpec = `file:${rel}`
         } else
-          updateDepSpec(pkg, name, req.saveSpec)
+          newSpec = req.saveSpec
+
+        if (options.saveType) {
+          const depType = saveTypeMap.get(options.saveType)
+          pkg[depType][name] = newSpec
+          // rpj will have moved it here if it was in both
+          // if it is empty it will be deleted later
+          if (options.saveType === 'prod' && pkg.optionalDependencies)
+            delete pkg.optionalDependencies[name]
+        } else {
+          if (hasSubKey(pkg, 'dependencies', name))
+            pkg.dependencies[name] = newSpec
+
+          if (hasSubKey(pkg, 'devDependencies', name)) {
+            pkg.devDependencies[name] = newSpec
+            // don't update peer or optional if we don't have to
+            if (hasSubKey(pkg, 'peerDependencies', name) && !intersects(newSpec, pkg.peerDependencies[name]))
+              pkg.peerDependencies[name] = newSpec
+
+            if (hasSubKey(pkg, 'optionalDependencies', name) && !intersects(newSpec, pkg.optionalDependencies[name]))
+              pkg.optionalDependencies[name] = newSpec
+          } else {
+            if (hasSubKey(pkg, 'peerDependencies', name))
+              pkg.peerDependencies[name] = newSpec
+
+            if (hasSubKey(pkg, 'optionalDependencies', name))
+              pkg.optionalDependencies[name] = newSpec
+          }
+        }
       }
 
       // refresh the edges so they have the correct specs

package/lib/audit-report.js

@@ -78,7 +78,7 @@ class AuditReport extends Map {
     }
 
     obj.vulnerabilities = vulnerabilities
-      .sort(([a], [b]) => a.localeCompare(b))
+      .sort(([a], [b]) => a.localeCompare(b, 'en'))
       .reduce((set, [name, vuln]) => {
         set[name] = vuln
         return set
@@ -101,13 +101,14 @@ class AuditReport extends Map {
 
   async run () {
     this.report = await this[_getReport]()
+    this.log.silly('audit report', this.report)
     if (this.report)
       await this[_init]()
     return this
   }
 
   isVulnerable (node) {
-    const vuln = this.get(node.package.name)
+    const vuln = this.get(node.packageName)
     return !!(vuln && vuln.isVulnerable(node))
   }
 
@@ -144,7 +145,7 @@ class AuditReport extends Map {
       super.set(name, vuln)
 
       const p = []
-      for (const node of this.tree.inventory.query('name', name)) {
+      for (const node of this.tree.inventory.query('packageName', name)) {
         if (shouldOmit(node, this[_omit]))
           continue
 
@@ -167,7 +168,7 @@ class AuditReport extends Map {
             this[_checkTopNode](dep, vuln, spec)
           else {
             // calculate a metavuln, if necessary
-            p.push(this.calculator.calculate(dep.name, advisory).then(meta => {
+            p.push(this.calculator.calculate(dep.packageName, advisory).then(meta => {
               if (meta.testVersion(dep.version, spec))
                 advisories.add(meta)
             }))
@@ -228,6 +229,9 @@ class AuditReport extends Map {
     if (!specObj.registry)
       return false
 
+    if (specObj.subSpec)
+      spec = specObj.subSpec.rawSpec
+
     // We don't provide fixes for top nodes other than root, but we
     // still check to see if the node is fixable with a different version,
     // and if that is a semver major bump.
@@ -289,6 +293,7 @@ class AuditReport extends Map {
       try {
         // first try the super fast bulk advisory listing
         const body = prepareBulkData(this.tree, this[_omit])
+        this.log.silly('audit', 'bulk request', body)
 
         // no sense asking if we don't have anything to audit,
         // we know it'll be empty
@@ -304,7 +309,8 @@ class AuditReport extends Map {
         })
 
         return await res.json()
-      } catch (_) {
+      } catch (er) {
+        this.log.silly('audit', 'bulk request failed', String(er.body))
         // that failed, try the quick audit endpoint
         const body = prepareData(this.tree, this.options)
         const res = await fetch('/-/npm/v1/security/audits/quick', {
@@ -330,6 +336,7 @@ class AuditReport extends Map {
 // return true if we should ignore this one
 const shouldOmit = (node, omit) =>
   !node.version ? true
+  : node.isRoot ? true
   : omit.size === 0 ? false
   : node.dev && omit.has('dev') ||
     node.optional && omit.has('optional') ||
@@ -338,9 +345,9 @@ const shouldOmit = (node, omit) =>
 
 const prepareBulkData = (tree, omit) => {
   const payload = {}
-  for (const name of tree.inventory.query('name')) {
+  for (const name of tree.inventory.query('packageName')) {
     const set = new Set()
-    for (const node of tree.inventory.query('name', name)) {
+    for (const node of tree.inventory.query('packageName', name)) {
       if (shouldOmit(node, omit))
         continue
 

package/lib/inventory.js

@@ -4,7 +4,7 @@
 // keys is the set of fields to be able to query.
 const _primaryKey = Symbol('_primaryKey')
 const _index = Symbol('_index')
-const defaultKeys = ['name', 'license', 'funding', 'realpath']
+const defaultKeys = ['name', 'license', 'funding', 'realpath', 'packageName']
 const { hasOwnProperty } = Object.prototype
 const debug = require('./debug.js')
 class Inventory extends Map {

package/lib/node.js

@@ -28,6 +28,7 @@
 // where we need to quickly find all instances of a given package name within a
 // tree.
 
+const semver = require('semver')
 const nameFromFolder = require('@npmcli/name-from-folder')
 const Edge = require('./edge.js')
 const Inventory = require('./inventory.js')
@@ -290,6 +291,10 @@ class Node {
     return this[_package].version || ''
   }
 
+  get packageName () {
+    return this[_package].name || null
+  }
+
   get pkgid () {
     const { name = '', version = '' } = this.package
     // root package will prefer package name over folder name,
@@ -349,10 +354,10 @@ class Node {
     }
 
     const why = {
-      name: this.isProjectRoot ? this.package.name : this.name,
+      name: this.isProjectRoot ? this.packageName : this.name,
       version: this.package.version,
     }
-    if (this.errors.length || !this.package.name || !this.package.version) {
+    if (this.errors.length || !this.packageName || !this.package.version) {
       why.errors = this.errors.length ? this.errors : [
         new Error('invalid package: lacks name and/or version'),
       ]
@@ -459,7 +464,7 @@ class Node {
     if (this.isProjectRoot)
       return false
     const { root } = this
-    const { type, to } = root.edgesOut.get(this.package.name) || {}
+    const { type, to } = root.edgesOut.get(this.packageName) || {}
     return type === 'workspace' && to && (to.target === this || to === this)
   }
 
@@ -729,20 +734,14 @@ class Node {
 
   [_loadDeps] () {
     // Caveat!  Order is relevant!
-    // packages in optionalDependencies and prod/peer/dev are
-    // optional.  Packages in both deps and devDeps are required.
+    // Packages in optionalDependencies are optional.
+    // Packages in both deps and devDeps are required.
     // Note the subtle breaking change from v6: it is no longer possible
     // to have a different spec for a devDep than production dep.
-    this[_loadDepType](this.package.optionalDependencies, 'optional')
 
     // Linked targets that are disconnected from the tree are tops,
     // but don't have a 'path' field, only a 'realpath', because we
     // don't know their canonical location. We don't need their devDeps.
-    const { isTop, path, sourceReference } = this
-    const { isTop: srcTop, path: srcPath } = sourceReference || {}
-    if (isTop && path && (!sourceReference || srcTop && srcPath))
-      this[_loadDepType](this.package.devDependencies, 'dev')
-
     const pd = this.package.peerDependencies
     if (pd && typeof pd === 'object' && !this.legacyPeerDeps) {
       const pm = this.package.peerDependenciesMeta || {}
@@ -759,19 +758,22 @@ class Node {
     }
 
     this[_loadDepType](this.package.dependencies, 'prod')
+    this[_loadDepType](this.package.optionalDependencies, 'optional')
+
+    const { isTop, path, sourceReference } = this
+    const { isTop: srcTop, path: srcPath } = sourceReference || {}
+    if (isTop && path && (!sourceReference || srcTop && srcPath))
+      this[_loadDepType](this.package.devDependencies, 'dev')
   }
 
-  [_loadDepType] (obj, type) {
-    const from = this
+  [_loadDepType] (deps, type) {
     const ad = this.package.acceptDependencies || {}
-    for (const [name, spec] of Object.entries(obj || {})) {
-      const accept = ad[name]
-      // if it's already set, then we keep the existing edge
-      // Prod deps should not be marked as dev, however.
-      // NB: the Edge ctor adds itself to from.edgesOut
+    // Because of the order in which _loadDeps runs, we always want to
+    // prioritize a new edge over an existing one
+    for (const [name, spec] of Object.entries(deps || {})) {
       const current = this.edgesOut.get(name)
-      if (!current || current.dev && type === 'prod')
-        new Edge({ from, name, spec, accept, type })
+      if (!current || current.type !== 'workspace')
+        new Edge({ from: this, name, spec, accept: ad[name], type })
     }
   }
 
@@ -885,6 +887,43 @@ class Node {
     return node.canReplaceWith(this)
   }
 
+  // return true if it's safe to remove this node, because anything that
+  // is depending on it would be fine with the thing that they would resolve
+  // to if it was removed, or nothing is depending on it in the first place.
+  canDedupe (preferDedupe = false) {
+    // not allowed to mess with shrinkwraps or bundles
+    if (this.inDepBundle || this.inShrinkwrap)
+      return false
+
+    // it's a top level pkg, or a dep of one
+    if (!this.parent || !this.parent.parent)
+      return false
+
+    // no one wants it, remove it
+    if (this.edgesIn.size === 0)
+      return true
+
+    const other = this.parent.parent.resolve(this.name)
+
+    // nothing else, need this one
+    if (!other)
+      return false
+
+    // if it's the same thing, then always fine to remove
+    if (other.matches(this))
+      return true
+
+    // if the other thing can't replace this, then skip it
+    if (!other.canReplace(this))
+      return false
+
+    // if we prefer dedupe, or if the version is greater/equal, take the other
+    if (preferDedupe || semver.gte(other.version, this.version))
+      return true
+
+    return false
+  }
+
   satisfies (requested) {
     if (requested instanceof Edge)
       return this.name === requested.name && requested.satisfiedBy(this)
@@ -927,8 +966,8 @@ class Node {
 
     // if no resolved, check both package name and version
     // otherwise, conclude that they are different things
-    return this.package.name && node.package.name &&
-      this.package.name === node.package.name &&
+    return this.packageName && node.packageName &&
+      this.packageName === node.packageName &&
       this.version && node.version &&
       this.version === node.version
   }

package/lib/printable.js

@@ -7,8 +7,8 @@ const relpath = require('./relpath.js')
 class ArboristNode {
   constructor (tree, path) {
     this.name = tree.name
-    if (tree.package.name && tree.package.name !== this.name)
-      this.packageName = tree.package.name
+    if (tree.packageName && tree.packageName !== this.name)
+      this.packageName = tree.packageName
     if (tree.version)
       this.version = tree.version
     this.location = tree.location
@@ -29,6 +29,15 @@ class ArboristNode {
       this.peer = true
     if (tree.inBundle)
       this.bundled = true
+    if (tree.inDepBundle)
+      this.bundler = tree.getBundler().location
+    const bd = tree.package && tree.package.bundleDependencies
+    if (bd && bd.length)
+      this.bundleDependencies = bd
+    if (tree.inShrinkwrap)
+      this.inShrinkwrap = true
+    else if (tree.hasShrinkwrap)
+      this.hasShrinkwrap = true
     if (tree.error)
       this.error = treeError(tree.error)
     if (tree.errors && tree.errors.length)
@@ -37,14 +46,14 @@ class ArboristNode {
     // edgesOut sorted by name
     if (tree.edgesOut.size) {
       this.edgesOut = new Map([...tree.edgesOut.entries()]
-        .sort(([a], [b]) => a.localeCompare(b))
+        .sort(([a], [b]) => a.localeCompare(b, 'en'))
         .map(([name, edge]) => [name, new EdgeOut(edge)]))
     }
 
     // edgesIn sorted by location
     if (tree.edgesIn.size) {
       this.edgesIn = new Set([...tree.edgesIn]
-        .sort((a, b) => a.from.location.localeCompare(b.from.location))
+        .sort((a, b) => a.from.location.localeCompare(b.from.location, 'en'))
         .map(edge => new EdgeIn(edge)))
     }
 
@@ -56,14 +65,14 @@ class ArboristNode {
     // fsChildren sorted by path
     if (tree.fsChildren.size) {
       this.fsChildren = new Set([...tree.fsChildren]
-        .sort(({path: a}, {path: b}) => a.localeCompare(b))
+        .sort(({path: a}, {path: b}) => a.localeCompare(b, 'en'))
         .map(tree => printableTree(tree, path)))
     }
 
     // children sorted by name
     if (tree.children.size) {
       this.children = new Map([...tree.children.entries()]
-        .sort(([a], [b]) => a.localeCompare(b))
+        .sort(([a], [b]) => a.localeCompare(b, 'en'))
         .map(([name, tree]) => [name, printableTree(tree, path)]))
     }
   }

package/lib/shrinkwrap.js

@@ -254,7 +254,7 @@ class Shrinkwrap {
         meta[key.replace(/^_/, '')] = val
     })
     // we only include name if different from the node path name
-    const pname = node.package.name
+    const pname = node.packageName
     if (pname && pname !== node.name)
       meta.name = pname
 
@@ -825,7 +825,7 @@ class Shrinkwrap {
   [_buildLegacyLockfile] (node, lock, path = []) {
     if (node === this.tree) {
       // the root node
-      lock.name = node.package.name || node.name
+      lock.name = node.packageName || node.name
       if (node.version)
         lock.version = node.version
     }
@@ -844,7 +844,7 @@ class Shrinkwrap {
       /* istanbul ignore next - sort calling order is indeterminate */
       return aloc.length > bloc.length ? 1
         : bloc.length > aloc.length ? -1
-        : aloc[aloc.length - 1].localeCompare(bloc[bloc.length - 1])
+        : aloc[aloc.length - 1].localeCompare(bloc[bloc.length - 1], 'en')
     })[0]
 
     const res = consistentResolve(node.resolved, this.path, this.path, true)
@@ -870,9 +870,9 @@ class Shrinkwrap {
         lock.from = spec.raw
     } else if (!node.isRoot &&
         node.package &&
-        node.package.name &&
-        node.package.name !== node.name)
-      lock.version = `npm:${node.package.name}@${node.version}`
+        node.packageName &&
+        node.packageName !== node.name)
+      lock.version = `npm:${node.packageName}@${node.version}`
     else if (node.package && node.version)
       lock.version = node.version
 

package/lib/update-root-package-json.js

@@ -6,8 +6,6 @@ const {resolve} = require('path')
 
 const parseJSON = require('json-parse-even-better-errors')
 
-const { orderDeps } = require('./dep-spec.js')
-
 const depTypes = new Set([
   'dependencies',
   'optionalDependencies',
@@ -15,6 +13,20 @@ const depTypes = new Set([
   'peerDependencies',
 ])
 
+// sort alphabetically all types of deps for a given package
+const orderDeps = (pkg) => {
+  for (const type of depTypes) {
+    if (pkg && pkg[type]) {
+      pkg[type] = Object.keys(pkg[type])
+        .sort((a, b) => a.localeCompare(b, 'en'))
+        .reduce((res, key) => {
+          res[key] = pkg[type][key]
+          return res
+        }, {})
+    }
+  }
+  return pkg
+}
 const parseJsonSafe = json => {
   try {
     return parseJSON(json)

package/lib/vuln.js

@@ -83,6 +83,9 @@ class Vuln {
     if (!specObj.registry)
       return true
 
+    if (specObj.subSpec)
+      spec = specObj.subSpec.rawSpec
+
     for (const v of this.versions) {
       if (satisfies(v, spec) && !satisfies(v, this.range, semverOpt))
         return false
@@ -103,12 +106,12 @@ class Vuln {
         vulnerableVersions: undefined,
         id: undefined,
       }).sort((a, b) =>
-        String(a.source || a).localeCompare(String(b.source || b))),
+        String(a.source || a).localeCompare(String(b.source || b, 'en'))),
       effects: [...this.effects].map(v => v.name)
-        .sort(/* istanbul ignore next */(a, b) => a.localeCompare(b)),
+        .sort(/* istanbul ignore next */(a, b) => a.localeCompare(b, 'en')),
       range: this.simpleRange,
       nodes: [...this.nodes].map(n => n.location)
-        .sort(/* istanbul ignore next */(a, b) => a.localeCompare(b)),
+        .sort(/* istanbul ignore next */(a, b) => a.localeCompare(b, 'en')),
       fixAvailable: this[_fixAvailable],
     }
   }

package/lib/yarn-lock.js

@@ -34,7 +34,7 @@ const {breadth} = require('treeverse')
 
 // sort a key/value object into a string of JSON stringified keys and vals
 const sortKV = obj => Object.keys(obj)
-  .sort((a, b) => a.localeCompare(b))
+  .sort((a, b) => a.localeCompare(b, 'en'))
   .map(k => `    ${JSON.stringify(k)} ${JSON.stringify(obj[k])}`)
   .join('\n')
 
@@ -165,7 +165,7 @@ class YarnLock {
   toString () {
     return prefix + [...new Set([...this.entries.values()])]
       .map(e => e.toString())
-      .sort((a, b) => a.localeCompare(b)).join('\n\n') + '\n'
+      .sort((a, b) => a.localeCompare(b, 'en')).join('\n\n') + '\n'
   }
 
   fromTree (tree) {
@@ -175,7 +175,7 @@ class YarnLock {
       tree,
       visit: node => this.addEntryFromNode(node),
       getChildren: node => [...node.children.values(), ...node.fsChildren]
-        .sort((a, b) => a.depth - b.depth || a.name.localeCompare(b.name)),
+        .sort((a, b) => a.depth - b.depth || a.name.localeCompare(b.name, 'en')),
     })
     return this
   }
@@ -183,7 +183,7 @@ class YarnLock {
   addEntryFromNode (node) {
     const specs = [...node.edgesIn]
       .map(e => `${node.name}@${e.spec}`)
-      .sort((a, b) => a.localeCompare(b))
+      .sort((a, b) => a.localeCompare(b, 'en'))
 
     // Note:
     // yarn will do excessive duplication in a case like this:
@@ -309,7 +309,7 @@ class YarnLockEntry {
   toString () {
     // sort objects to the bottom, then alphabetical
     return ([...this[_specs]]
-      .sort((a, b) => a.localeCompare(b))
+      .sort((a, b) => a.localeCompare(b, 'en'))
       .map(JSON.stringify).join(', ') +
       ':\n' +
       Object.getOwnPropertyNames(this)
@@ -318,7 +318,7 @@ class YarnLockEntry {
           (a, b) =>
           /* istanbul ignore next - sort call order is unpredictable */
             (typeof this[a] === 'object') === (typeof this[b] === 'object')
-              ? a.localeCompare(b)
+              ? a.localeCompare(b, 'en')
               : typeof this[a] === 'object' ? 1 : -1)
         .map(prop =>
           typeof this[prop] !== 'object'

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@npmcli/arborist",
-  "version": "2.4.0",
+  "version": "2.4.4",
   "description": "Manage node_modules trees",
   "dependencies": {
     "@npmcli/installed-package-contents": "^1.0.7",
@@ -14,7 +14,7 @@
     "cacache": "^15.0.3",
     "common-ancestor-path": "^1.0.1",
     "json-parse-even-better-errors": "^2.3.1",
-    "json-stringify-nice": "^1.1.2",
+    "json-stringify-nice": "^1.1.4",
     "mkdirp-infer-owner": "^2.0.0",
     "npm-install-checks": "^4.0.0",
     "npm-package-arg": "^8.1.0",
@@ -40,9 +40,8 @@
     "eslint-plugin-promise": "^4.2.1",
     "eslint-plugin-standard": "^4.0.1",
     "minify-registry-metadata": "^2.1.0",
-    "mutate-fs": "^2.1.1",
-    "tap": "^15.0.4",
-    "tcompare": "^3.0.4"
+    "tap": "^15.0.9",
+    "tcompare": "^5.0.6"
   },
   "scripts": {
     "test": "npm run test-only --",
@@ -74,11 +73,13 @@
   "bin": {
     "arborist": "bin/index.js"
   },
+  "//": "sk test-env locale to catch locale-specific sorting",
   "tap": {
     "after": "test/fixtures/cleanup.js",
     "coverage-map": "map.js",
     "test-env": [
-      "NODE_OPTIONS=--no-warnings"
+      "NODE_OPTIONS=--no-warnings",
+      "LC_ALL=sk"
     ],
     "node-arg": [
       "--no-warnings",

package/CHANGELOG.md

@@ -1,19 +0,0 @@
-# CHANGELOG
-
-## 2.0
-
-* BREAKING CHANGE: root node is now included in inventory
-* All parent/target/fsParent/etc. references set in `root` setter, rather
-  than the hodgepodge of setters that existed before.
-* `treeCheck` function added, to enforce strict correctness guarantees when
-  `ARBORIST_DEBUG=1` in the environment (on by default in Arborist tests).
-
-## 1.0
-
-* Release for npm v7 beta
-* Fully functional
-
-## 0.0
-
-* Proof of concept
-* Before this, it was [`read-package-tree`](http://npm.im/read-package-tree)

package/lib/dep-spec.js

@@ -1,43 +0,0 @@
-const types = [
-  'peerDependencies',
-  'devDependencies',
-  'optionalDependencies',
-  'dependencies',
-]
-
-const findType = (pkg, name) => {
-  for (const t of types) {
-    if (pkg[t] && typeof pkg[t] === 'object' && pkg[t][name] !== undefined)
-      return t
-  }
-  return 'dependencies'
-}
-
-// given a dep name and spec, update it wherever it exists in
-// the manifest, or add the spec to 'dependencies' if not found.
-const updateDepSpec = (pkg, name, newSpec) => {
-  const type = findType(pkg, name)
-  pkg[type] = pkg[type] || {}
-  pkg[type][name] = newSpec
-  return pkg
-}
-
-// sort alphabetically all types of deps for a given package
-const orderDeps = (pkg) => {
-  for (const type of types) {
-    if (pkg && pkg[type]) {
-      pkg[type] = Object.keys(pkg[type])
-        .sort((a, b) => a.localeCompare(b))
-        .reduce((res, key) => {
-          res[key] = pkg[type][key]
-          return res
-        }, {})
-    }
-  }
-  return pkg
-}
-
-module.exports = {
-  orderDeps,
-  updateDepSpec,
-}