@npmcli/arborist 2.1.0 → 2.2.2

package/bin/actual.js

@@ -0,0 +1,21 @@
+const Arborist = require('../')
+const print = require('./lib/print-tree.js')
+const options = require('./lib/options.js')
+require('./lib/logging.js')
+require('./lib/timers.js')
+
+const start = process.hrtime()
+new Arborist(options).loadActual(options).then(tree => {
+  const end = process.hrtime(start)
+  if (!process.argv.includes('--quiet'))
+    print(tree)
+
+  console.error(`read ${tree.inventory.size} deps in ${end[0] * 1000 + end[1] / 1e6}ms`)
+  if (options.save)
+    tree.meta.save()
+  if (options.saveHidden) {
+    tree.meta.hiddenLockfile = true
+    tree.meta.filename = options.path + '/node_modules/.package-lock.json'
+    tree.meta.save()
+  }
+}).catch(er => console.error(er))

package/bin/audit.js

@@ -0,0 +1,48 @@
+const Arborist = require('../')
+
+const print = require('./lib/print-tree.js')
+const options = require('./lib/options.js')
+require('./lib/timers.js')
+require('./lib/logging.js')
+
+const Vuln = require('../lib/vuln.js')
+const printReport = report => {
+  for (const vuln of report.values())
+    console.log(printVuln(vuln))
+  if (report.topVulns.size) {
+    console.log('\n# top-level vulnerabilities')
+    for (const vuln of report.topVulns.values())
+      console.log(printVuln(vuln))
+  }
+}
+
+const printVuln = vuln => {
+  return {
+    __proto__: { constructor: Vuln },
+    name: vuln.name,
+    issues: [...vuln.advisories].map(a => printAdvisory(a)),
+    range: vuln.simpleRange,
+    nodes: [...vuln.nodes].map(node => `${node.name} ${node.location || '#ROOT'}`),
+    ...(vuln.topNodes.size === 0 ? {} : {
+      topNodes: [...vuln.topNodes].map(node => `${node.location || '#ROOT'}`),
+    }),
+  }
+}
+
+const printAdvisory = a => `${a.title}${a.url ? ' ' + a.url : ''}`
+
+const start = process.hrtime()
+process.emit('time', 'audit script')
+const arb = new Arborist(options)
+arb.audit(options).then(tree => {
+  process.emit('timeEnd', 'audit script')
+  const end = process.hrtime(start)
+  if (options.fix)
+    print(tree)
+  if (!options.quiet)
+    printReport(arb.auditReport)
+  if (options.fix)
+    console.error(`resolved ${tree.inventory.size} deps in ${end[0] + end[1] / 1e9}s`)
+  if (tree.meta && options.save)
+    tree.meta.save()
+}).catch(er => console.error(er))

package/bin/funding.js

@@ -0,0 +1,32 @@
+const options = require('./lib/options.js')
+require('./lib/logging.js')
+require('./lib/timers.js')
+
+const Arborist = require('../')
+const a = new Arborist(options)
+const query = options._.shift()
+const start = process.hrtime()
+a.loadVirtual().then(tree => {
+  // only load the actual tree if the virtual one doesn't have modern metadata
+  if (!tree.meta || !(tree.meta.originalLockfileVersion >= 2)) {
+    console.error('old metadata, load actual')
+    throw 'load actual'
+  } else {
+    console.error('meta ok, return virtual tree')
+    return tree
+  }
+}).catch(() => a.loadActual()).then(tree => {
+  const end = process.hrtime(start)
+  if (!query) {
+    for (const node of tree.inventory.values()) {
+      if (node.package.funding)
+        console.log(node.name, node.location, node.package.funding)
+    }
+  } else {
+    for (const node of tree.inventory.query('name', query)) {
+      if (node.package.funding)
+        console.log(node.name, node.location, node.package.funding)
+    }
+  }
+  console.error(`read ${tree.inventory.size} deps in ${end[0] * 1000 + end[1] / 1e6}ms`)
+})

package/bin/ideal.js

@@ -0,0 +1,20 @@
+const Arborist = require('../')
+
+const { inspect } = require('util')
+const options = require('./lib/options.js')
+const print = require('./lib/print-tree.js')
+require('./lib/logging.js')
+require('./lib/timers.js')
+
+const start = process.hrtime()
+new Arborist(options).buildIdealTree(options).then(tree => {
+  const end = process.hrtime(start)
+  print(tree)
+  console.error(`resolved ${tree.inventory.size} deps in ${end[0] + end[1] / 10e9}s`)
+  if (tree.meta && options.save)
+    tree.meta.save()
+}).catch(er => {
+  const opt = { depth: Infinity, color: true }
+  console.error(er.code === 'ERESOLVE' ? inspect(er, opt) : er)
+  process.exitCode = 1
+})

package/bin/index.js

@@ -0,0 +1,77 @@
+#!/usr/bin/env node
+const [cmd] = process.argv.splice(2, 1)
+
+const usage = () => `Arborist - the npm tree doctor
+
+Version: ${require('../package.json').version}
+
+# USAGE
+  arborist <cmd> [path] [options...]
+
+# COMMANDS
+
+* reify: reify ideal tree to node_modules (install, update, rm, ...)
+* ideal: generate and print the ideal tree
+* actual: read and print the actual tree in node_modules
+* virtual: read and print the virtual tree in the local shrinkwrap file
+* shrinkwrap: load a local shrinkwrap and print its data
+* audit: perform a security audit on project dependencies
+* funding: query funding information in the local package tree.  A second
+  positional argument after the path name can limit to a package name.
+* license: query license information in the local package tree.  A second
+  positional argument after the path name can limit to a license type.
+* help: print this text
+
+# OPTIONS
+
+Most npm options are supported, but in camelCase rather than css-case.  For
+example, instead of '--dry-run', use '--dryRun'.
+
+Additionally:
+
+* --quiet will supppress the printing of package trees
+* Instead of 'npm install <pkg>', use 'arborist reify --add=<pkg>'.
+  The '--add=<pkg>' option can be specified multiple times.
+* Instead of 'npm rm <pkg>', use 'arborist reify --rm=<pkg>'.
+  The '--rm=<pkg>' option can be specified multiple times.
+* Instead of 'npm update', use 'arborist reify --update-all'.
+* 'npm audit fix' is 'arborist audit --fix'
+`
+
+const help = () => console.log(usage())
+
+switch (cmd) {
+  case 'actual':
+    require('./actual.js')
+    break
+  case 'virtual':
+    require('./virtual.js')
+    break
+  case 'ideal':
+    require('./ideal.js')
+    break
+  case 'reify':
+    require('./reify.js')
+    break
+  case 'audit':
+    require('./audit.js')
+    break
+  case 'funding':
+    require('./funding.js')
+    break
+  case 'license':
+    require('./license.js')
+    break
+  case 'shrinkwrap':
+    require('./shrinkwrap.js')
+    break
+  case 'help':
+  case '-h':
+  case '--help':
+    help()
+    break
+  default:
+    process.exitCode = 1
+    console.error(usage())
+    break
+}

package/bin/lib/logging.js

@@ -0,0 +1,33 @@
+const options = require('./options.js')
+const { quiet = false } = options
+const { loglevel = quiet ? 'warn' : 'silly' } = options
+
+const levels = [
+  'silly',
+  'verbose',
+  'info',
+  'timing',
+  'http',
+  'notice',
+  'warn',
+  'error',
+  'silent',
+]
+
+const levelMap = new Map(levels.reduce((set, level, index) => {
+  set.push([level, index], [index, level])
+  return set
+}, []))
+
+const { inspect, format } = require('util')
+if (loglevel !== 'silent') {
+  process.on('log', (level, ...args) => {
+    if (levelMap.get(level) < levelMap.get(loglevel))
+      return
+    const pref = `${process.pid} ${level} `
+    if (level === 'warn' && args[0] === 'ERESOLVE')
+      args[2] = inspect(args[2], { depth: 10 })
+    const msg = pref + format(...args).trim().split('\n').join(`\n${pref}`)
+    console.error(msg)
+  })
+}

package/bin/lib/options.js

@@ -0,0 +1,49 @@
+const options = module.exports = {
+  path: undefined,
+  cache: `${process.env.HOME}/.npm/_cacache`,
+  _: [],
+}
+
+for (const arg of process.argv.slice(2)) {
+  if (/^--add=/.test(arg)) {
+    options.add = options.add || []
+    options.add.push(arg.substr('--add='.length))
+  } else if (/^--rm=/.test(arg)) {
+    options.rm = options.rm || []
+    options.rm.push(arg.substr('--rm='.length))
+  } else if (arg === '--global')
+    options.global = true
+  else if (arg === '--global-style')
+    options.globalStyle = true
+  else if (arg === '--prefer-dedupe')
+    options.preferDedupe = true
+  else if (arg === '--legacy-peer-deps')
+    options.legacyPeerDeps = true
+  else if (arg === '--force')
+    options.force = true
+  else if (arg === '--update-all') {
+    options.update = options.update || {}
+    options.update.all = true
+  } else if (/^--update=/.test(arg)) {
+    options.update = options.update || {}
+    options.update.names = options.update.names || []
+    options.update.names.push(arg.substr('--update='.length))
+  } else if (/^--omit=/.test(arg)) {
+    options.omit = options.omit || []
+    options.omit.push(arg.substr('--omit='.length))
+  } else if (/^--[^=]+=/.test(arg)) {
+    const [key, ...v] = arg.replace(/^--/, '').split('=')
+    const val = v.join('=')
+    options[key] = val === 'false' ? false : val === 'true' ? true : val
+  } else if (/^--.+/.test(arg))
+    options[arg.replace(/^--/, '')] = true
+  else if (options.path === undefined)
+    options.path = arg
+  else
+    options._.push(arg)
+}
+
+if (options.path === undefined)
+  options.path = '.'
+
+console.error(options)

package/bin/lib/print-tree.js

@@ -0,0 +1,5 @@
+const { inspect } = require('util')
+const { quiet } = require('./options.js')
+
+module.exports = quiet ? () => {}
+  : tree => console.log(inspect(tree.toJSON(), { depth: Infinity }))

package/bin/lib/timers.js

@@ -0,0 +1,22 @@
+const timers = Object.create(null)
+
+process.on('time', name => {
+  if (timers[name])
+    throw new Error('conflicting timer! ' + name)
+  timers[name] = process.hrtime()
+})
+
+process.on('timeEnd', name => {
+  if (!timers[name])
+    throw new Error('timer not started! ' + name)
+  const res = process.hrtime(timers[name])
+  delete timers[name]
+  console.error(`${process.pid} ${name}`, res[0] * 1e3 + res[1] / 1e6)
+})
+
+process.on('exit', () => {
+  for (const name of Object.keys(timers)) {
+    console.error('Dangling timer: ', name)
+    process.exitCode = 1
+  }
+})

package/bin/license.js

@@ -0,0 +1,34 @@
+const Arborist = require('../')
+const options = require('./lib/options.js')
+require('./lib/logging.js')
+require('./lib/timers.js')
+
+const a = new Arborist(options)
+const query = options._.shift()
+
+a.loadVirtual().then(tree => {
+  // only load the actual tree if the virtual one doesn't have modern metadata
+  if (!tree.meta || !(tree.meta.originalLockfileVersion >= 2))
+    throw 'load actual'
+  else
+    return tree
+}).catch((er) => {
+  console.error('loading actual tree', er)
+  return a.loadActual()
+}).then(tree => {
+  if (!query) {
+    const set = []
+    for (const license of tree.inventory.query('license'))
+      set.push([tree.inventory.query('license', license).size, license])
+
+    for (const [count, license] of set.sort((a, b) =>
+      a[1] && b[1] ? b[0] - a[0] || a[1].localeCompare(b[1])
+      : a[1] ? -1
+      : b[1] ? 1
+      : 0))
+      console.log(count, license)
+  } else {
+    for (const node of tree.inventory.query('license', query === 'undefined' ? undefined : query))
+      console.log(`${node.name} ${node.location} ${node.package.description || ''}`)
+  }
+})

package/bin/reify.js

@@ -0,0 +1,46 @@
+const Arborist = require('../')
+
+const options = require('./lib/options.js')
+const print = require('./lib/print-tree.js')
+require('./lib/logging.js')
+require('./lib/timers.js')
+
+const printDiff = diff => {
+  const {depth} = require('treeverse')
+  depth({
+    tree: diff,
+    visit: d => {
+      if (d.location === '')
+        return
+      switch (d.action) {
+        case 'REMOVE':
+          console.error('REMOVE', d.actual.location)
+          break
+        case 'ADD':
+          console.error('ADD', d.ideal.location, d.ideal.resolved)
+          break
+        case 'CHANGE':
+          console.error('CHANGE', d.actual.location, {
+            from: d.actual.resolved,
+            to: d.ideal.resolved,
+          })
+          break
+      }
+    },
+    getChildren: d => d.children,
+  })
+}
+
+const start = process.hrtime()
+process.emit('time', 'install')
+const arb = new Arborist(options)
+arb.reify(options).then(tree => {
+  process.emit('timeEnd', 'install')
+  const end = process.hrtime(start)
+  print(tree)
+  if (options.dryRun)
+    printDiff(arb.diff)
+  console.error(`resolved ${tree.inventory.size} deps in ${end[0] + end[1] / 1e9}s`)
+  if (tree.meta && options.save)
+    tree.meta.save()
+}).catch(er => console.error(require('util').inspect(er, { depth: Infinity })))

package/bin/shrinkwrap.js

@@ -0,0 +1,12 @@
+const Shrinkwrap = require('../lib/shrinkwrap.js')
+const options = require('./lib/options.js')
+require('./lib/logging.js')
+require('./lib/timers.js')
+
+const { quiet } = options
+Shrinkwrap.load(options)
+  .then(s => quiet || console.log(JSON.stringify(s.data, 0, 2)))
+  .catch(er => {
+    console.error('shrinkwrap load failure', er)
+    process.exit(1)
+  })

package/bin/virtual.js

@@ -0,0 +1,15 @@
+const Arborist = require('../')
+
+const print = require('./lib/print-tree.js')
+const options = require('./lib/options.js')
+require('./lib/logging.js')
+require('./lib/timers.js')
+
+const start = process.hrtime()
+new Arborist(options).loadVirtual().then(tree => {
+  const end = process.hrtime(start)
+  print(tree)
+  if (options.save)
+    tree.meta.save()
+  console.error(`read ${tree.inventory.size} deps in ${end[0] * 1000 + end[1] / 1e6}ms`)
+}).catch(er => console.error(er))

package/lib/arborist/build-ideal-tree.js

@@ -397,7 +397,8 @@ module.exports = cls => class IdealTreeBuilder extends cls {
     // that they're there, and not reinstall the world unnecessarily.
     if (this[_global] && (this[_updateAll] || this[_updateNames].length)) {
       const nm = resolve(this.path, 'node_modules')
-      for (const name of await readdir(nm)) {
+      for (const name of await readdir(nm).catch(() => [])) {
+        tree.package.dependencies = tree.package.dependencies || {}
         if (this[_updateAll] || this[_updateNames].includes(name))
           tree.package.dependencies[name] = '*'
       }
@@ -416,7 +417,7 @@ module.exports = cls => class IdealTreeBuilder extends cls {
       await this[_add](options)
 
     // triggers a refresh of all edgesOut
-    if (options.add && options.add.length || options.rm && options.rm.length)
+    if (options.add && options.add.length || options.rm && options.rm.length || this[_global])
       tree.package = tree.package
     process.emit('timeEnd', 'idealTree:userRequests')
   }
@@ -490,7 +491,8 @@ module.exports = cls => class IdealTreeBuilder extends cls {
     /* istanbul ignore else - should also be covered by realpath failure */
     if (filepath) {
       const { name } = spec
-      spec = npa(`file:${relpath(this.path, filepath)}`, this.path)
+      const tree = this.idealTree.target || this.idealTree
+      spec = npa(`file:${relpath(tree.path, filepath)}`, tree.path)
       spec.name = name
     }
     return spec
@@ -662,6 +664,11 @@ This is a one-time fix-up, please be patient...
       })
     }
     await promiseCallLimit(queue)
+
+    // have to re-calc dep flags, because the nodes don't have edges
+    // until their packages get assigned, so everything looks extraneous
+    calcDepFlags(this.idealTree)
+
     // yes, yes, this isn't the "original" version, but now that it's been
     // upgraded, we need to make sure we don't do the work to upgrade it
     // again, since it's now as new as can be.
@@ -799,6 +806,7 @@ This is a one-time fix-up, please be patient...
       // a virtual root of whatever brought in THIS node.
       // so we VR the node itself if the edge is not a peer
       const source = edge.peer ? peerSource : node
+
       const virtualRoot = this[_virtualRoot](source, true)
       // reuse virtual root if we already have one, but don't
       // try to do the override ahead of time, since we MAY be able
@@ -820,13 +828,17 @@ This is a one-time fix-up, please be patient...
       // +-- z@1
       // But if x and y are loaded in the same virtual root, then they will
       // be forced to agree on a version of z.
+      const required = new Set([edge.from])
+      const parent = edge.peer ? virtualRoot : null
       const dep = vrDep && vrDep.satisfies(edge) ? vrDep
-        : await this[_nodeFromEdge](edge, edge.peer ? virtualRoot : null)
+        : await this[_nodeFromEdge](edge, parent, null, required)
+
       /* istanbul ignore next */
       debug(() => {
         if (!dep)
           throw new Error('no dep??')
       })
+
       tasks.push({edge, dep})
     }
 
@@ -863,7 +875,7 @@ This is a one-time fix-up, please be patient...
 
   // loads a node from an edge, and then loads its peer deps (and their
   // peer deps, on down the line) into a virtual root parent.
-  async [_nodeFromEdge] (edge, parent_, secondEdge = null) {
+  async [_nodeFromEdge] (edge, parent_, secondEdge, required) {
     // create a virtual root node with the same deps as the node that
     // is requesting this one, so that we can get all the peer deps in
     // a context where they're likely to be resolvable.
@@ -894,6 +906,11 @@ This is a one-time fix-up, please be patient...
     // ensure the one we want is the one that's placed
     node.parent = parent
 
+    if (required.has(edge.from) && edge.type !== 'peerOptional' ||
+        secondEdge && (
+          required.has(secondEdge.from) && secondEdge.type !== 'peerOptional'))
+      required.add(node)
+
     // handle otherwise unresolvable dependency nesting loops by
     // creating a symbolic link
     // a1 -> b1 -> a2 -> b2 -> a1 -> ...
@@ -907,7 +924,7 @@ This is a one-time fix-up, please be patient...
     // keep track of the thing that caused this node to be included.
     const src = parent.sourceReference
     this[_peerSetSource].set(node, src)
-    return this[_loadPeerSet](node)
+    return this[_loadPeerSet](node, required)
   }
 
   [_virtualRoot] (node, reuse = false) {
@@ -1052,7 +1069,7 @@ This is a one-time fix-up, please be patient...
   // gets placed first.  In non-strict mode, we behave strictly if the
   // virtual root is based on the root project, and allow non-peer parent
   // deps to override, but throw if no preference can be determined.
-  async [_loadPeerSet] (node) {
+  async [_loadPeerSet] (node, required) {
     const peerEdges = [...node.edgesOut.values()]
       // we typically only install non-optional peers, but we have to
       // factor them into the peerSet so that we can avoid conflicts
@@ -1067,10 +1084,12 @@ This is a one-time fix-up, please be patient...
       const parentEdge = node.parent.edgesOut.get(edge.name)
       const {isProjectRoot, isWorkspace} = node.parent.sourceReference
       const isMine = isProjectRoot || isWorkspace
+      const conflictOK = this[_force] || !isMine && !this[_strictPeerDeps]
+
       if (!edge.to) {
         if (!parentEdge) {
           // easy, just put the thing there
-          await this[_nodeFromEdge](edge, node.parent)
+          await this[_nodeFromEdge](edge, node.parent, null, required)
           continue
         } else {
           // if the parent's edge is very broad like >=1, and the edge in
@@ -1081,14 +1100,16 @@ This is a one-time fix-up, please be patient...
           // a conflict.  this is always a problem in strict mode, never
           // in force mode, and a problem in non-strict mode if this isn't
           // on behalf of our project.  in all such cases, we warn at least.
-          await this[_nodeFromEdge](parentEdge, node.parent, edge)
+          const dep = await this[_nodeFromEdge](parentEdge, node.parent, edge, required)
 
           // hooray! that worked!
           if (edge.valid)
             continue
 
-          // allow it
-          if (this[_force] || !isMine && !this[_strictPeerDeps])
+          // allow it.  either we're overriding, or it's not something
+          // that will be installed by default anyway, and we'll fail when
+          // we get to the point where we need to, if we need to.
+          if (conflictOK || !required.has(dep))
             continue
 
           // problem
@@ -1101,7 +1122,7 @@ This is a one-time fix-up, please be patient...
       // in non-strict mode if it's not our fault.  don't warn here, because
       // we are going to warn again when we place the deps, if we end up
       // overriding for something else.
-      if (this[_force] || !isMine && !this[_strictPeerDeps])
+      if (conflictOK)
         continue
 
       // ok, it's the root, or we're in unforced strict mode, so this is bad
@@ -1148,6 +1169,7 @@ This is a one-time fix-up, please be patient...
   [_placeDep] (dep, node, edge, peerEntryEdge = null, peerPath = []) {
     if (edge.to &&
         !edge.error &&
+        !this[_explicitRequests].has(edge.name) &&
         !this[_updateNames].includes(edge.name) &&
         !this[_isVulnerable](edge.to))
       return []
@@ -1196,8 +1218,25 @@ This is a one-time fix-up, please be patient...
         break
     }
 
-    if (!target)
-      this[_failPeerConflict](edge)
+    // if we can't find a target, that means that the last placed checked
+    // (and all the places before it) had a copy already.  if we're in
+    // --force mode, then the user has explicitly said that they're ok
+    // with conflicts.  This can only occur in --force mode in the case
+    // when a node was added to the tree with a peerOptional dep that we
+    // ignored, and then later, that edge became invalid, and we fail to
+    // resolve it.  We will warn about it in a moment.
+    if (!target) {
+      if (this[_force]) {
+        // we know that there is a dep (not the root) which is the target
+        // of this edge, or else it wouldn't have been a conflict.
+        target = edge.to.resolveParent
+        canPlace = KEEP
+      } else
+        this[_failPeerConflict](edge)
+    } else {
+      // it worked, so we clearly have no peer conflicts at this point.
+      this[_peerConflict] = null
+    }
 
     this.log.silly(
       'placeDep',
@@ -1208,9 +1247,6 @@ This is a one-time fix-up, please be patient...
       `want: ${edge.spec || '*'}`
     )
 
-    // it worked, so we clearly have no peer conflicts at this point.
-    this[_peerConflict] = null
-
     // Can only get KEEP here if the original edge was valid,
     // and we're checking for an update but it's already up to date.
     if (canPlace === KEEP) {
@@ -1396,6 +1432,7 @@ This is a one-time fix-up, please be patient...
     })
     const entryEdge = peerEntryEdge || edge
     const source = this[_peerSetSource].get(dep)
+
     isSource = isSource || target === source
     // if we're overriding the source, then we care if the *target* is
     // ours, even if it wasn't actually the original source, since we

package/lib/arborist/load-virtual.js

@@ -24,6 +24,7 @@ const loadWorkspacesVirtual = Symbol.for('loadWorkspacesVirtual')
 const flagsSuspect = Symbol.for('flagsSuspect')
 const reCalcDepFlags = Symbol('reCalcDepFlags')
 const checkRootEdges = Symbol('checkRootEdges')
+const rootOptionProvided = Symbol('rootOptionProvided')
 
 const depsToEdges = (type, deps) =>
   Object.entries(deps).map(d => [type, ...d])
@@ -63,6 +64,8 @@ module.exports = cls => class VirtualLoader extends cls {
       root = await this[loadRoot](s),
     } = options
 
+    this[rootOptionProvided] = options.root
+
     await this[loadFromShrinkwrap](s, root)
     return treeCheck(this.virtualTree)
   }
@@ -74,13 +77,17 @@ module.exports = cls => class VirtualLoader extends cls {
   }
 
   async [loadFromShrinkwrap] (s, root) {
-    // root is never any of these things, but might be a brand new
-    // baby Node object that never had its dep flags calculated.
-    root.extraneous = false
-    root.dev = false
-    root.optional = false
-    root.devOptional = false
-    root.peer = false
+    if (!this[rootOptionProvided]) {
+      // root is never any of these things, but might be a brand new
+      // baby Node object that never had its dep flags calculated.
+      root.extraneous = false
+      root.dev = false
+      root.optional = false
+      root.devOptional = false
+      root.peer = false
+    } else
+      this[flagsSuspect] = true
+
     this[checkRootEdges](s, root)
     root.meta = s
     this.virtualTree = root
@@ -88,20 +95,23 @@ module.exports = cls => class VirtualLoader extends cls {
     await this[resolveLinks](links, nodes)
     this[assignBundles](nodes)
     if (this[flagsSuspect])
-      this[reCalcDepFlags]()
+      this[reCalcDepFlags](nodes.values())
     return root
   }
 
-  [reCalcDepFlags] () {
+  [reCalcDepFlags] (nodes) {
     // reset all dep flags
-    for (const node of this.virtualTree.inventory.values()) {
+    // can't use inventory here, because virtualTree might not be root
+    for (const node of nodes) {
+      if (node.isRoot || node === this[rootOptionProvided])
+        continue
       node.extraneous = true
       node.dev = true
       node.optional = true
       node.devOptional = true
       node.peer = true
     }
-    calcDepFlags(this.virtualTree, true)
+    calcDepFlags(this.virtualTree, !this[rootOptionProvided])
   }
 
   // check the lockfile deps, and see if they match.  if they do not
@@ -237,6 +247,12 @@ module.exports = cls => class VirtualLoader extends cls {
     // shrinkwrap doesn't include package name unless necessary
     if (!sw.name)
       sw.name = nameFromFolder(path)
+
+    const dev = sw.dev
+    const optional = sw.optional
+    const devOptional = dev || optional || sw.devOptional
+    const peer = sw.peer
+
     const node = new Node({
       legacyPeerDeps: this.legacyPeerDeps,
       root: this.virtualTree,
@@ -246,6 +262,10 @@ module.exports = cls => class VirtualLoader extends cls {
       resolved: consistentResolve(sw.resolved, this.path, path),
       pkg: sw,
       hasShrinkwrap: sw.hasShrinkwrap,
+      dev,
+      optional,
+      devOptional,
+      peer,
     })
     // cast to boolean because they're undefined in the lock file when false
     node.extraneous = !!sw.extraneous

package/lib/arborist/reify.js

@@ -233,7 +233,7 @@ module.exports = cls => class Reifier extends cls {
     const actualOpt = this[_global] ? {
       ignoreMissing: true,
       global: true,
-      filter: (node, kid) => !node.isRoot && node !== node.root.target
+      filter: (node, kid) => this[_explicitRequests].size === 0 || !node.isProjectRoot
         ? true
         : (node.edgesOut.has(kid) || this[_explicitRequests].has(kid)),
     } : { ignoreMissing: true }
@@ -442,7 +442,8 @@ module.exports = cls => class Reifier extends cls {
     if (this[_trashList].has(node.path))
       return node
 
-    process.emit('time', `reifyNode:${node.location}`)
+    const timer = `reifyNode:${node.location}`
+    process.emit('time', timer)
     this.addTracker('reify', node.name, node.location)
 
     const p = Promise.resolve()
@@ -454,7 +455,7 @@ module.exports = cls => class Reifier extends cls {
     return this[_handleOptionalFailure](node, p)
       .then(() => {
         this.finishTracker('reify', node.name, node.location)
-        process.emit('timeEnd', `reifyNode:${node.location}`)
+        process.emit('timeEnd', timer)
         return node
       })
   }
@@ -474,9 +475,14 @@ module.exports = cls => class Reifier extends cls {
 
     // no idea what this thing is.  remove it from the tree.
     if (!res) {
-      node.parent = null
+      const warning = 'invalid or damaged lockfile detected\n' +
+        'please re-try this operation once it completes\n' +
+        'so that the damage can be corrected, or perform\n' +
+        'a fresh install with no lockfile if the problem persists.'
+      this.log.warn('reify', warning)
       this.log.verbose('reify', 'unrecognized node in tree', node.path)
       node.parent = null
+      node.fsParent = null
       this[_addNodeToTrashList](node)
       return
     }
@@ -605,7 +611,7 @@ module.exports = cls => class Reifier extends cls {
       tree: this.diff,
       visit: diff => {
         const node = diff.ideal
-        if (node && !node.isRoot && node.package.bundleDependencies &&
+        if (node && !node.isProjectRoot && node.package.bundleDependencies &&
             node.package.bundleDependencies.length) {
           maxBundleDepth = Math.max(maxBundleDepth, node.depth)
           if (!bundlesByDepth.has(node.depth))
@@ -811,7 +817,7 @@ module.exports = cls => class Reifier extends cls {
     dfwalk({
       tree: this.diff,
       leave: diff => {
-        if (!diff.ideal.isRoot)
+        if (!diff.ideal.isProjectRoot)
           nodes.push(diff.ideal)
       },
       // process adds before changes, ignore removals
@@ -907,7 +913,7 @@ module.exports = cls => class Reifier extends cls {
 
     return Promise.all([
       this[_saveLockFile](saveOpt),
-      updateRootPackageJson({ tree: this.idealTree }),
+      updateRootPackageJson(this.idealTree),
     ]).then(() => process.emit('timeEnd', 'reify:save'))
   }
 

package/lib/calc-dep-flags.js

@@ -11,7 +11,7 @@ const calcDepFlags = (tree, resetRoot = true) => {
     tree,
     visit: node => calcDepFlagsStep(node),
     filter: node => node,
-    getChildren: node => [...node.edgesOut.values()].map(edge => edge.to),
+    getChildren: (node, tree) => [...tree.edgesOut.values()].map(edge => edge.to),
   })
   return ret
 }

package/lib/node.js

@@ -247,7 +247,7 @@ class Node {
 
   // true for packages installed directly in the global node_modules folder
   get globalTop () {
-    return this.global && this.parent.isRoot
+    return this.global && this.parent.isProjectRoot
   }
 
   get workspaces () {
@@ -294,8 +294,11 @@ class Node {
     const { name = '', version = '' } = this.package
     // root package will prefer package name over folder name,
     // and never be called an alias.
-    const myname = this.isRoot ? name || this.name : this.name
-    const alias = !this.isRoot && name && myname !== name ? `npm:${name}@` : ''
+    const { isProjectRoot } = this
+    const myname = isProjectRoot ? name || this.name
+      : this.name
+    const alias = !isProjectRoot && name && myname !== name ? `npm:${name}@`
+      : ''
     return `${myname}@${alias}${version}`
   }
 
@@ -339,14 +342,14 @@ class Node {
   }
 
   [_explain] (edge, seen) {
-    if (this.isRoot && !this.sourceReference) {
+    if (this.isProjectRoot && !this.sourceReference) {
       return {
         location: this.path,
       }
     }
 
     const why = {
-      name: this.isRoot ? this.package.name : this.name,
+      name: this.isProjectRoot ? this.package.name : this.name,
       version: this.package.version,
     }
     if (this.errors.length || !this.package.name || !this.package.version) {
@@ -384,7 +387,7 @@ class Node {
       // and are not keeping it held in this spot anyway.
       const edges = []
       for (const edge of this.edgesIn) {
-        if (!edge.valid && !edge.from.isRoot)
+        if (!edge.valid && !edge.from.isProjectRoot)
           continue
 
         edges.push(edge)
@@ -453,7 +456,7 @@ class Node {
   }
 
   get isWorkspace () {
-    if (this.isRoot)
+    if (this.isProjectRoot)
       return false
     const { root } = this
     const { type, to } = root.edgesOut.get(this.package.name) || {}
@@ -733,7 +736,9 @@ class Node {
     // Linked targets that are disconnected from the tree are tops,
     // but don't have a 'path' field, only a 'realpath', because we
     // don't know their canonical location. We don't need their devDeps.
-    if (this.isTop && this.path && !this.sourceReference)
+    const { isTop, path, sourceReference } = this
+    const { isTop: srcTop, path: srcPath } = sourceReference || {}
+    if (isTop && path && (!sourceReference || srcTop && srcPath))
       this[_loadDepType](this.package.devDependencies, 'dev')
 
     const pd = this.package.peerDependencies
@@ -902,8 +907,8 @@ class Node {
     if (this.isLink)
       return node.isLink && this.target.matches(node.target)
 
-    // if they're two root nodes, they're different if the paths differ
-    if (this.isRoot && node.isRoot)
+    // if they're two project root nodes, they're different if the paths differ
+    if (this.isProjectRoot && node.isProjectRoot)
       return this.path === node.path
 
     // if the integrity matches, then they're the same.

package/lib/update-root-package-json.js

@@ -15,7 +15,7 @@ const depTypes = new Set([
   'peerDependencies',
 ])
 
-async function updateRootPackageJson ({ tree }) {
+const updateRootPackageJson = async tree => {
   const filename = resolve(tree.path, 'package.json')
   const originalContent = await readFile(filename, 'utf8')
     .then(data => parseJSON(data))
@@ -25,6 +25,16 @@ async function updateRootPackageJson ({ tree }) {
     ...tree.package,
   })
 
+  // optionalDependencies don't need to be repeated in two places
+  if (depsData.dependencies) {
+    if (depsData.optionalDependencies) {
+      for (const name of Object.keys(depsData.optionalDependencies))
+        delete depsData.dependencies[name]
+    }
+    if (Object.keys(depsData.dependencies).length === 0)
+      delete depsData.dependencies
+  }
+
   // if there's no package.json, just use internal pkg info as source of truth
   const packageJsonContent = originalContent || depsData
 

package/package.json

@@ -1,15 +1,15 @@
 {
   "name": "@npmcli/arborist",
-  "version": "2.1.0",
+  "version": "2.2.2",
   "description": "Manage node_modules trees",
   "dependencies": {
-    "@npmcli/installed-package-contents": "^1.0.5",
-    "@npmcli/map-workspaces": "^1.0.1",
+    "@npmcli/installed-package-contents": "^1.0.6",
+    "@npmcli/map-workspaces": "^1.0.2",
     "@npmcli/metavuln-calculator": "^1.0.1",
     "@npmcli/move-file": "^1.1.0",
     "@npmcli/name-from-folder": "^1.0.1",
     "@npmcli/node-gyp": "^1.0.1",
-    "@npmcli/run-script": "^1.8.1",
+    "@npmcli/run-script": "^1.8.2",
     "bin-links": "^2.2.1",
     "cacache": "^15.0.3",
     "common-ancestor-path": "^1.0.1",
@@ -20,11 +20,11 @@
     "npm-package-arg": "^8.1.0",
     "npm-pick-manifest": "^6.1.0",
     "npm-registry-fetch": "^9.0.0",
-    "pacote": "^11.2.3",
+    "pacote": "^11.2.6",
     "parse-conflict-json": "^1.1.1",
     "promise-all-reject-late": "^1.0.0",
     "promise-call-limit": "^1.0.1",
-    "read-package-json-fast": "^1.2.1",
+    "read-package-json-fast": "^2.0.1",
     "readdir-scoped-modules": "^1.1.0",
     "semver": "^7.3.4",
     "tar": "^6.1.0",
@@ -55,7 +55,7 @@
     "postversion": "npm publish",
     "prepublishOnly": "git push origin --follow-tags",
     "eslint": "eslint",
-    "lint": "npm run eslint -- \"lib/**/*.js\"",
+    "lint": "npm run eslint -- \"lib/**/*.js\" \"test/arborist/*.js\" \"test/*.js\" \"bin/**/*.js\"",
     "lintfix": "npm run lint -- --fix",
     "benchmark": "node scripts/benchmark.js",
     "benchclean": "rm -rf scripts/benchmark/*/"
@@ -67,9 +67,13 @@
   "author": "Isaac Z. Schlueter <i@izs.me> (http://blog.izs.me/)",
   "license": "ISC",
   "files": [
-    "lib/**/*.js"
+    "lib/**/*.js",
+    "bin/**/*.js"
   ],
   "main": "lib/index.js",
+  "bin": {
+    "arborist": "bin/index.js"
+  },
   "tap": {
     "100": true,
     "node-arg": [