@npmcli/arborist 2.7.1 → 2.8.3

package/lib/arborist/reify.js

@@ -5,11 +5,14 @@ const pacote = require('pacote')
 const AuditReport = require('../audit-report.js')
 const {subset, intersects} = require('semver')
 const npa = require('npm-package-arg')
+const debug = require('../debug.js')
+const walkUp = require('walk-up-path')
 
 const {dirname, resolve, relative} = require('path')
 const {depth: dfwalk} = require('treeverse')
 const fs = require('fs')
 const {promisify} = require('util')
+const lstat = promisify(fs.lstat)
 const symlink = promisify(fs.symlink)
 const mkdirp = require('mkdirp-infer-owner')
 const justMkdirp = require('mkdirp')
@@ -18,6 +21,7 @@ const rimraf = promisify(require('rimraf'))
 const PackageJson = require('@npmcli/package-json')
 const packageContents = require('@npmcli/installed-package-contents')
 const { checkEngine, checkPlatform } = require('npm-install-checks')
+const _force = Symbol.for('force')
 
 const treeCheck = require('../tree-check.js')
 const relpath = require('../relpath.js')
@@ -50,6 +54,7 @@ const _createSparseTree = Symbol.for('createSparseTree')
 const _loadShrinkwrapsAndUpdateTrees = Symbol.for('loadShrinkwrapsAndUpdateTrees')
 const _shrinkwrapInflated = Symbol('shrinkwrapInflated')
 const _bundleUnpacked = Symbol('bundleUnpacked')
+const _bundleMissing = Symbol('bundleMissing')
 const _reifyNode = Symbol.for('reifyNode')
 const _extractOrLink = Symbol('extractOrLink')
 // defined by rebuild mixin
@@ -74,8 +79,10 @@ const _copyIdealToActual = Symbol('copyIdealToActual')
 const _addOmitsToTrashList = Symbol('addOmitsToTrashList')
 const _packageLockOnly = Symbol('packageLockOnly')
 const _dryRun = Symbol('dryRun')
+const _validateNodeModules = Symbol('validateNodeModules')
+const _nmValidated = Symbol('nmValidated')
 const _validatePath = Symbol('validatePath')
-const _reifyPackages = Symbol('reifyPackages')
+const _reifyPackages = Symbol.for('reifyPackages')
 
 const _omitDev = Symbol('omitDev')
 const _omitOptional = Symbol('omitOptional')
@@ -83,8 +90,9 @@ const _omitPeer = Symbol('omitPeer')
 
 const _global = Symbol.for('global')
 
+const _pruneBundledMetadeps = Symbol('pruneBundledMetadeps')
+
 // defined by Ideal mixin
-const _pruneBundledMetadeps = Symbol.for('pruneBundledMetadeps')
 const _resolvedAdd = Symbol.for('resolvedAdd')
 const _usePackageLock = Symbol.for('usePackageLock')
 const _formatPackageLock = Symbol.for('formatPackageLock')
@@ -112,6 +120,11 @@ module.exports = cls => class Reifier extends cls {
     this[_sparseTreeDirs] = new Set()
     this[_sparseTreeRoots] = new Set()
     this[_trashList] = new Set()
+    // the nodes we unpack to read their bundles
+    this[_bundleUnpacked] = new Set()
+    // child nodes we'd EXPECT to be included in a bundle, but aren't
+    this[_bundleMissing] = new Set()
+    this[_nmValidated] = new Set()
   }
 
   // public method
@@ -145,19 +158,24 @@ module.exports = cls => class Reifier extends cls {
 
   async [_validatePath] () {
     // don't create missing dirs on dry runs
-    if (this[_packageLockOnly] || this[_dryRun])
+    if (this[_packageLockOnly] || this[_dryRun]) {
       return
+    }
 
     // we do NOT want to set ownership on this folder, especially
     // recursively, because it can have other side effects to do that
     // in a project directory.  We just want to make it if it's missing.
     await justMkdirp(resolve(this.path))
+
+    // do not allow the top-level node_modules to be a symlink
+    await this[_validateNodeModules](resolve(this.path, 'node_modules'))
   }
 
   async [_reifyPackages] () {
     // we don't submit the audit report or write to disk on dry runs
-    if (this[_dryRun])
+    if (this[_dryRun]) {
       return
+    }
 
     if (this[_packageLockOnly]) {
       // we already have the complete tree, so just audit it now,
@@ -204,8 +222,9 @@ module.exports = cls => class Reifier extends cls {
       for (const action of actions) {
         try {
           await this[action]()
-          if (reifyTerminated)
+          if (reifyTerminated) {
             throw reifyTerminated
+          }
         } catch (er) {
           await this[rollback](er)
           /* istanbul ignore next - rollback throws, should never hit this */
@@ -217,8 +236,9 @@ module.exports = cls => class Reifier extends cls {
     // no rollback for this one, just exit with the error, since the
     // install completed and can't be safely recovered at this point.
     await this[_removeTrash]()
-    if (reifyTerminated)
+    if (reifyTerminated) {
       throw reifyTerminated
+    }
 
     // done modifying the file system, no need to keep listening for sigs
     removeHandler()
@@ -245,18 +265,21 @@ module.exports = cls => class Reifier extends cls {
       filter: (node, kid) => {
         // if it's not the project root, and we have no explicit requests,
         // then we're already into a nested dep, so we keep it
-        if (this.explicitRequests.size === 0 || !node.isProjectRoot)
+        if (this.explicitRequests.size === 0 || !node.isProjectRoot) {
           return true
+        }
 
         // if we added it as an edgeOut, then we want it
-        if (this.idealTree.edgesOut.has(kid))
+        if (this.idealTree.edgesOut.has(kid)) {
           return true
+        }
 
         // if it's an explicit request, then we want it
         const hasExplicit = [...this.explicitRequests]
           .some(edge => edge.name === kid)
-        if (hasExplicit)
+        if (hasExplicit) {
           return true
+        }
 
         // ignore the rest of the global install folder
         return false
@@ -264,8 +287,10 @@ module.exports = cls => class Reifier extends cls {
     } : { ignoreMissing: true }
 
     if (!this[_global]) {
-      return Promise.all([this.loadActual(actualOpt), this.buildIdealTree(bitOpt)])
-        .then(() => process.emit('timeEnd', 'reify:loadTrees'))
+      return Promise.all([
+        this.loadActual(actualOpt),
+        this.buildIdealTree(bitOpt),
+      ]).then(() => process.emit('timeEnd', 'reify:loadTrees'))
     }
 
     // the global install space tends to have a lot of stuff in it.  don't
@@ -279,8 +304,9 @@ module.exports = cls => class Reifier extends cls {
   }
 
   [_diffTrees] () {
-    if (this[_packageLockOnly])
+    if (this[_packageLockOnly]) {
       return
+    }
 
     process.emit('time', 'reify:diffTrees')
     // XXX if we have an existing diff already, there should be a way
@@ -295,20 +321,24 @@ module.exports = cls => class Reifier extends cls {
       // children where there's an explicit request.
       for (const { name } of this.explicitRequests) {
         const ideal = idealTree.children.get(name)
-        if (ideal)
+        if (ideal) {
           filterNodes.push(ideal)
+        }
         const actual = actualTree.children.get(name)
-        if (actual)
+        if (actual) {
           filterNodes.push(actual)
+        }
       }
     } else {
       for (const ws of this[_workspaces]) {
         const ideal = this.idealTree.children.get(ws)
-        if (ideal)
+        if (ideal) {
           filterNodes.push(ideal)
+        }
         const actual = this.actualTree.children.get(ws)
-        if (actual)
+        if (actual) {
           filterNodes.push(actual)
+        }
       }
     }
 
@@ -321,12 +351,13 @@ module.exports = cls => class Reifier extends cls {
       ideal: this.idealTree,
     })
 
-    for (const node of this.diff.removed) {
-      // a node in a dep bundle will only be removed if its bundling dep
-      // is removed as well.  in which case, we don't have to delete it!
-      if (!node.inDepBundle)
-        this[_addNodeToTrashList](node)
-    }
+    // we don't have to add 'removed' folders to the trashlist, because
+    // they'll be moved aside to a retirement folder, and then the retired
+    // folder will be deleted at the end.  This is important when we have
+    // a folder like FOO being "removed" in favor of a folder like "foo",
+    // because if we remove node_modules/FOO on case-insensitive systems,
+    // it will remove the dep that we *want* at node_modules/foo.
+
     process.emit('timeEnd', 'reify:diffTrees')
   }
 
@@ -334,7 +365,7 @@ module.exports = cls => class Reifier extends cls {
   // removed later on in the process.  optionally, also mark them
   // as a retired paths, so that we move them out of the way and
   // replace them when rolling back on failure.
-  [_addNodeToTrashList] (node, retire) {
+  [_addNodeToTrashList] (node, retire = false) {
     const paths = [node.path, ...node.binPaths]
     const moves = this[_retiredPaths]
     this.log.silly('reify', 'mark', retire ? 'retired' : 'deleted', paths)
@@ -343,8 +374,9 @@ module.exports = cls => class Reifier extends cls {
         const retired = retirePath(path)
         moves[path] = retired
         this[_trashList].add(retired)
-      } else
+      } else {
         this[_trashList].add(path)
+      }
     }
   }
 
@@ -376,10 +408,11 @@ module.exports = cls => class Reifier extends cls {
         if (er.code === 'ENOENT') {
           return didMkdirp ? null : mkdirp(dirname(to)).then(() =>
             this[_renamePath](from, to, true))
-        } else if (er.code === 'EEXIST')
+        } else if (er.code === 'EEXIST') {
           return rimraf(to).then(() => moveFile(from, to))
-        else
+        } else {
           throw er
+        }
       })
   }
 
@@ -400,8 +433,9 @@ module.exports = cls => class Reifier extends cls {
   // adding to the trash list will skip reifying, and delete them
   // if they are currently in the tree and otherwise untouched.
   [_addOmitsToTrashList] () {
-    if (!this[_omitDev] && !this[_omitOptional] && !this[_omitPeer])
+    if (!this[_omitDev] && !this[_omitOptional] && !this[_omitPeer]) {
       return
+    }
 
     process.emit('time', 'reify:trashOmits')
 
@@ -412,8 +446,9 @@ module.exports = cls => class Reifier extends cls {
           node.optional && this[_omitOptional] ||
           node.devOptional && this[_omitOptional] && this[_omitDev])
 
-    for (const node of this.idealTree.inventory.filter(filter))
+    for (const node of this.idealTree.inventory.filter(filter)) {
       this[_addNodeToTrashList](node)
+    }
 
     process.emit('timeEnd', 'reify:trashOmits')
   }
@@ -422,19 +457,42 @@ module.exports = cls => class Reifier extends cls {
     process.emit('time', 'reify:createSparse')
     // if we call this fn again, we look for the previous list
     // so that we can avoid making the same directory multiple times
-    const dirs = this.diff.leaves
+    const leaves = this.diff.leaves
       .filter(diff => {
         return (diff.action === 'ADD' || diff.action === 'CHANGE') &&
           !this[_sparseTreeDirs].has(diff.ideal.path) &&
           !diff.ideal.isLink
       })
-      .map(diff => diff.ideal.path)
-
-    return promiseAllRejectLate(dirs.map(d => mkdirp(d)))
-      .then(made => {
-        made.forEach(made => this[_sparseTreeRoots].add(made))
-        dirs.forEach(dir => this[_sparseTreeDirs].add(dir))
-      })
+      .map(diff => diff.ideal)
+
+    // we check this in parallel, so guard against multiple attempts to
+    // retire the same path at the same time.
+    const dirsChecked = new Set()
+    return promiseAllRejectLate(leaves.map(async node => {
+      for (const d of walkUp(node.path)) {
+        if (d === node.top.path) {
+          break
+        }
+        if (dirsChecked.has(d)) {
+          continue
+        }
+        dirsChecked.add(d)
+        const st = await lstat(d).catch(er => null)
+        // this can happen if we have a link to a package with a name
+        // that the filesystem treats as if it is the same thing.
+        // would be nice to have conditional istanbul ignores here...
+        /* istanbul ignore next - defense in depth */
+        if (st && !st.isDirectory()) {
+          const retired = retirePath(d)
+          this[_retiredPaths][d] = retired
+          this[_trashList].add(retired)
+          await this[_renamePath](d, retired)
+        }
+      }
+      const made = await mkdirp(node.path)
+      this[_sparseTreeDirs].add(node.path)
+      this[_sparseTreeRoots].add(made)
+    }))
       .then(() => process.emit('timeEnd', 'reify:createSparse'))
   }
 
@@ -449,8 +507,9 @@ module.exports = cls => class Reifier extends cls {
       .map(path => rimraf(path).catch(er => failures.push([path, er])))
     return promiseAllRejectLate(unlinks)
       .then(() => {
-        if (failures.length)
+        if (failures.length) {
           this.log.warn('cleanup', 'Failed to remove some directories', failures)
+        }
       })
       .then(() => process.emit('timeEnd', 'reify:rollback:createSparse'))
       .then(() => this[_rollbackRetireShallowNodes](er))
@@ -466,8 +525,9 @@ module.exports = cls => class Reifier extends cls {
         d.ideal.hasShrinkwrap && !seen.has(d.ideal) &&
         !this[_trashList].has(d.ideal.path))
 
-    if (!shrinkwraps.length)
+    if (!shrinkwraps.length) {
       return
+    }
 
     process.emit('time', 'reify:loadShrinkwraps')
 
@@ -497,8 +557,9 @@ module.exports = cls => class Reifier extends cls {
   // to the trash list
   // Always return the node.
   [_reifyNode] (node) {
-    if (this[_trashList].has(node.path))
+    if (this[_trashList].has(node.path)) {
       return node
+    }
 
     const timer = `reifyNode:${node.location}`
     process.emit('time', timer)
@@ -529,7 +590,21 @@ module.exports = cls => class Reifier extends cls {
       })
   }
 
-  [_extractOrLink] (node) {
+  // do not allow node_modules to be a symlink
+  async [_validateNodeModules] (nm) {
+    if (this[_force] || this[_nmValidated].has(nm)) {
+      return
+    }
+    const st = await lstat(nm).catch(() => null)
+    if (!st || st.isDirectory()) {
+      this[_nmValidated].add(nm)
+      return
+    }
+    this.log.warn('reify', 'Removing non-directory', nm)
+    await rimraf(nm)
+  }
+
+  async [_extractOrLink] (node) {
     // in normal cases, node.resolved should *always* be set by now.
     // however, it is possible when a lockfile is damaged, or very old,
     // or in some other race condition bugs in npm v6, that a previously
@@ -556,13 +631,29 @@ module.exports = cls => class Reifier extends cls {
       return
     }
 
-    return node.isLink
-      ? rimraf(node.path).then(() => this[_symlink](node))
-      : pacote.extract(res, node.path, {
+    const nm = resolve(node.parent.path, 'node_modules')
+    await this[_validateNodeModules](nm)
+
+    if (node.isLink) {
+      await rimraf(node.path)
+      await this[_symlink](node)
+    } else {
+      await debug(async () => {
+        const st = await lstat(node.path).catch(e => null)
+        if (st && !st.isDirectory()) {
+          debug.log('unpacking into a non-directory', node)
+          throw Object.assign(new Error('ENOTDIR: not a directory'), {
+            code: 'ENOTDIR',
+            path: node.path,
+          })
+        }
+      })
+      await pacote.extract(res, node.path, {
         ...this.options,
         resolved: node.resolved,
         integrity: node.integrity,
       })
+    }
   }
 
   async [_symlink] (node) {
@@ -575,8 +666,9 @@ module.exports = cls => class Reifier extends cls {
 
   [_warnDeprecated] (node) {
     const {_id, deprecated} = node.package
-    if (deprecated)
+    if (deprecated) {
       this.log.warn('deprecated', `${_id}: ${deprecated}`)
+    }
   }
 
   // if the node is optional, then the failure of the promise is nonfatal
@@ -611,9 +703,9 @@ module.exports = cls => class Reifier extends cls {
     depth = 0, bundlesByDepth = this[_getBundlesByDepth]()
   ) {
     if (depth === 0) {
-      this[_bundleUnpacked] = new Set()
       process.emit('time', 'reify:loadBundles')
     }
+
     const maxBundleDepth = bundlesByDepth.get('maxBundleDepth')
     if (depth > maxBundleDepth) {
       // if we did something, then prune the tree and update the diffs
@@ -632,8 +724,9 @@ module.exports = cls => class Reifier extends cls {
         node.target !== node.root &&
         !this[_trashList].has(node.path))
 
-    if (!set.length)
+    if (!set.length) {
       return this[_loadBundlesAndUpdateTrees](depth + 1, bundlesByDepth)
+    }
 
     // extract all the nodes with bundles
     return promiseAllRejectLate(set.map(node => {
@@ -642,14 +735,32 @@ module.exports = cls => class Reifier extends cls {
     }))
     // then load their unpacked children and move into the ideal tree
       .then(nodes =>
-        promiseAllRejectLate(nodes.map(node => new this.constructor({
-          ...this.options,
-          path: node.path,
-        }).loadActual({
-          root: node,
-          // don't transplant any sparse folders we created
-          transplantFilter: node => node.package._id,
-        }))))
+        promiseAllRejectLate(nodes.map(async node => {
+          const arb = new this.constructor({
+            ...this.options,
+            path: node.path,
+          })
+          const notTransplanted = new Set(node.children.keys())
+          await arb.loadActual({
+            root: node,
+            // don't transplant any sparse folders we created
+            // loadActual will set node.package to {} for empty directories
+            // if by chance there are some empty folders in the node_modules
+            // tree for some other reason, then ok, ignore those too.
+            transplantFilter: node => {
+              if (node.package._id) {
+                // it's actually in the bundle if it gets transplanted
+                notTransplanted.delete(node.name)
+                return true
+              } else {
+                return false
+              }
+            },
+          })
+          for (const name of notTransplanted) {
+            this[_bundleMissing].add(node.children.get(name))
+          }
+        })))
     // move onto the next level of bundled items
       .then(() => this[_loadBundlesAndUpdateTrees](depth + 1, bundlesByDepth))
   }
@@ -661,18 +772,21 @@ module.exports = cls => class Reifier extends cls {
       tree: this.diff,
       visit: diff => {
         const node = diff.ideal
-        if (!node)
+        if (!node) {
           return
-        if (node.isProjectRoot)
+        }
+        if (node.isProjectRoot) {
           return
+        }
 
         const { bundleDependencies } = node.package
         if (bundleDependencies && bundleDependencies.length) {
           maxBundleDepth = Math.max(maxBundleDepth, node.depth)
-          if (!bundlesByDepth.has(node.depth))
+          if (!bundlesByDepth.has(node.depth)) {
             bundlesByDepth.set(node.depth, [node])
-          else
+          } else {
             bundlesByDepth.get(node.depth).push(node)
+          }
         }
       },
       getChildren: diff => diff.children,
@@ -685,57 +799,95 @@ module.exports = cls => class Reifier extends cls {
   // https://github.com/npm/cli/issues/1597#issuecomment-667639545
   [_pruneBundledMetadeps] (bundlesByDepth) {
     const bundleShadowed = new Set()
+
+    // Example dep graph:
+    // root -> (a, c)
+    // a -> BUNDLE(b)
+    // b -> c
+    // c -> b
+    //
+    // package tree:
+    // root
+    // +-- a
+    // |   +-- b(1)
+    // |   +-- c(1)
+    // +-- b(2)
+    // +-- c(2)
+    // 1. mark everything that's shadowed by anything in the bundle.  This
+    //    marks b(2) and c(2).
+    // 2. anything with edgesIn from outside the set, mark not-extraneous,
+    //    remove from set.  This unmarks c(2).
+    // 3. continue until no change
+    // 4. remove everything in the set from the tree.  b(2) is pruned
+
     // create the list of nodes shadowed by children of bundlers
     for (const bundles of bundlesByDepth.values()) {
       // skip the 'maxBundleDepth' item
-      if (!Array.isArray(bundles))
+      if (!Array.isArray(bundles)) {
         continue
+      }
       for (const node of bundles) {
         for (const name of node.children.keys()) {
           const shadow = node.parent.resolve(name)
-          if (!shadow)
+          if (!shadow) {
             continue
+          }
           bundleShadowed.add(shadow)
           shadow.extraneous = true
         }
       }
     }
-    let changed = true
-    while (changed) {
-      changed = false
-      for (const shadow of bundleShadowed) {
-        if (!shadow.extraneous) {
-          bundleShadowed.delete(shadow)
-          continue
+
+    // lib -> (a@1.x) BUNDLE(a@1.2.3 (b@1.2.3))
+    // a@1.2.3 -> (b@1.2.3)
+    // a@1.3.0 -> (b@2)
+    // b@1.2.3 -> ()
+    // b@2 -> (c@2)
+    //
+    // root
+    // +-- lib
+    // |   +-- a@1.2.3
+    // |   +-- b@1.2.3
+    // +-- b@2 <-- shadowed, now extraneous
+    // +-- c@2 <-- also shadowed, because only dependent is shadowed
+    for (const shadow of bundleShadowed) {
+      for (const shadDep of shadow.edgesOut.values()) {
+        /* istanbul ignore else - pretty unusual situation, just being
+         * defensive here. Would mean that a bundled dep has a dependency
+         * that is unmet. which, weird, but if you bundle it, we take
+         * whatever you put there and assume the publisher knows best. */
+        if (shadDep.to) {
+          bundleShadowed.add(shadDep.to)
+          shadDep.to.extraneous = true
         }
+      }
+    }
 
+    let changed
+    do {
+      changed = false
+      for (const shadow of bundleShadowed) {
         for (const edge of shadow.edgesIn) {
-          if (!edge.from.extraneous) {
+          if (!bundleShadowed.has(edge.from)) {
             shadow.extraneous = false
             bundleShadowed.delete(shadow)
             changed = true
-          } else {
-            for (const shadDep of shadow.edgesOut.values()) {
-              /* istanbul ignore else - pretty unusual situation, just being
-               * defensive here. Would mean that a bundled dep has a dependency
-               * that is unmet. which, weird, but if you bundle it, we take
-               * whatever you put there and assume the publisher knows best. */
-              if (shadDep.to)
-                bundleShadowed.add(shadDep.to)
-            }
+            break
           }
         }
       }
-    }
+    } while (changed)
+
     for (const shadow of bundleShadowed) {
-      shadow.parent = null
       this[_addNodeToTrashList](shadow)
+      shadow.root = null
     }
   }
 
   [_submitQuickAudit] () {
-    if (this.options.audit === false)
+    if (this.options.audit === false) {
       return this.auditReport = null
+    }
 
     // we submit the quick audit at this point in the process, as soon as
     // we have all the deps resolved, so that it can overlap with the other
@@ -748,8 +900,9 @@ module.exports = cls => class Reifier extends cls {
     const tree = this.idealTree
 
     // if we're operating on a workspace, only audit the workspace deps
-    if (this[_workspaces] && this[_workspaces].length)
+    if (this[_workspaces] && this[_workspaces].length) {
       options.filterSet = this.workspaceDependencySet(tree, this[_workspaces])
+    }
 
     this.auditReport = AuditReport.load(tree, options)
       .then(res => {
@@ -774,23 +927,30 @@ module.exports = cls => class Reifier extends cls {
       tree: this.diff,
       visit: diff => {
         // no unpacking if we don't want to change this thing
-        if (diff.action !== 'CHANGE' && diff.action !== 'ADD')
+        if (diff.action !== 'CHANGE' && diff.action !== 'ADD') {
           return
+        }
 
         const node = diff.ideal
         const bd = this[_bundleUnpacked].has(node)
         const sw = this[_shrinkwrapInflated].has(node)
+        const bundleMissing = this[_bundleMissing].has(node)
 
         // check whether we still need to unpack this one.
         // test the inDepBundle last, since that's potentially a tree walk.
         const doUnpack = node && // can't unpack if removed!
-          !node.isRoot && // root node already exists
-          !bd && // already unpacked to read bundle
-          !sw && // already unpacked to read sw
-          !node.inDepBundle // already unpacked by another dep's bundle
-
-        if (doUnpack)
+          // root node already exists
+          !node.isRoot &&
+          // already unpacked to read bundle
+          !bd &&
+          // already unpacked to read sw
+          !sw &&
+          // already unpacked by another dep's bundle
+          (bundleMissing || !node.inDepBundle)
+
+        if (doUnpack) {
           unpacks.push(this[_reifyNode](node))
+        }
       },
       getChildren: diff => diff.children,
     })
@@ -814,17 +974,38 @@ module.exports = cls => class Reifier extends cls {
     const moves = this[_retiredPaths]
     this[_retiredUnchanged] = {}
     return promiseAllRejectLate(this.diff.children.map(diff => {
-      const realFolder = (diff.actual || diff.ideal).path
+      // skip if nothing was retired
+      if (diff.action !== 'CHANGE' && diff.action !== 'REMOVE') {
+        return
+      }
+
+      const { path: realFolder } = diff.actual
       const retireFolder = moves[realFolder]
+      /* istanbul ignore next - should be impossible */
+      debug(() => {
+        if (!retireFolder) {
+          const er = new Error('trying to un-retire but not retired')
+          throw Object.assign(er, {
+            realFolder,
+            retireFolder,
+            actual: diff.actual,
+            ideal: diff.ideal,
+            action: diff.action,
+          })
+        }
+      })
+
       this[_retiredUnchanged][retireFolder] = []
       return promiseAllRejectLate(diff.unchanged.map(node => {
         // no need to roll back links, since we'll just delete them anyway
-        if (node.isLink)
+        if (node.isLink) {
           return mkdirp(dirname(node.path)).then(() => this[_reifyNode](node))
+        }
 
         // will have been moved/unpacked along with bundler
-        if (node.inDepBundle)
+        if (node.inDepBundle && !this[_bundleMissing].has(node)) {
           return
+        }
 
         this[_retiredUnchanged][retireFolder].push(node)
 
@@ -878,8 +1059,9 @@ module.exports = cls => class Reifier extends cls {
     dfwalk({
       tree: this.diff,
       leave: diff => {
-        if (!diff.ideal.isProjectRoot)
+        if (!diff.ideal.isProjectRoot) {
           nodes.push(diff.ideal)
+        }
       },
       // process adds before changes, ignore removals
       getChildren: diff => diff && diff.children,
@@ -894,8 +1076,9 @@ module.exports = cls => class Reifier extends cls {
       // skip links that only live within node_modules as they are most
       // likely managed by packages we installed, we only want to rebuild
       // unchanged links we directly manage
-      if (node.isLink && node.target.fsTop === tree)
+      if (node.isLink && node.target.fsTop === tree) {
         nodes.push(node)
+      }
     }
 
     return this.rebuild({ nodes, handleOptionalFailure: true })
@@ -912,12 +1095,14 @@ module.exports = cls => class Reifier extends cls {
     const failures = []
     const rm = path => rimraf(path).catch(er => failures.push([path, er]))
 
-    for (const path of this[_trashList])
+    for (const path of this[_trashList]) {
       promises.push(rm(path))
+    }
 
     return promiseAllRejectLate(promises).then(() => {
-      if (failures.length)
+      if (failures.length) {
         this.log.warn('cleanup', 'Failed to remove some directories', failures)
+      }
     })
       .then(() => process.emit('timeEnd', 'reify:trash'))
   }
@@ -931,8 +1116,9 @@ module.exports = cls => class Reifier extends cls {
     // save it first, then prune out the optional trash, and then return it.
 
     // support save=false option
-    if (options.save === false || this[_global] || this[_dryRun])
+    if (options.save === false || this[_global] || this[_dryRun]) {
       return false
+    }
 
     process.emit('time', 'reify:save')
 
@@ -953,6 +1139,14 @@ module.exports = cls => class Reifier extends cls {
         const spec = subSpec ? subSpec.rawSpec : rawSpec
         const child = edge.to
 
+        // if we tried to install an optional dep, but it was a version
+        // that we couldn't resolve, this MAY be missing.  if we haven't
+        // blown up by now, it's because it was not a problem, though, so
+        // just move on.
+        if (!child) {
+          continue
+        }
+
         let newSpec
         if (req.registry) {
           const version = child.version
@@ -969,8 +1163,9 @@ module.exports = cls => class Reifier extends cls {
             !isRange ||
             spec === '*' ||
             subset(prefixRange, spec, { loose: true })
-          )
+          ) {
             range = prefixRange
+          }
 
           const pname = child.packageName
           const alias = name !== pname
@@ -979,10 +1174,11 @@ module.exports = cls => class Reifier extends cls {
           // save the git+https url if it has auth, otherwise shortcut
           const h = req.hosted
           const opt = { noCommittish: false }
-          if (h.https && h.auth)
+          if (h.https && h.auth) {
             newSpec = `git+${h.https(opt)}`
-          else
+          } else {
             newSpec = h.shortcut(opt)
+          }
         } else if (req.type === 'directory' || req.type === 'file') {
           // save the relative path in package.json
           // Normally saveSpec is updated with the proper relative
@@ -992,34 +1188,41 @@ module.exports = cls => class Reifier extends cls {
           const p = req.fetchSpec.replace(/^file:/, '')
           const rel = relpath(addTree.realpath, p)
           newSpec = `file:${rel}`
-        } else
+        } else {
           newSpec = req.saveSpec
+        }
 
         if (options.saveType) {
           const depType = saveTypeMap.get(options.saveType)
           pkg[depType][name] = newSpec
           // rpj will have moved it here if it was in both
           // if it is empty it will be deleted later
-          if (options.saveType === 'prod' && pkg.optionalDependencies)
+          if (options.saveType === 'prod' && pkg.optionalDependencies) {
             delete pkg.optionalDependencies[name]
+          }
         } else {
-          if (hasSubKey(pkg, 'dependencies', name))
+          if (hasSubKey(pkg, 'dependencies', name)) {
             pkg.dependencies[name] = newSpec
+          }
 
           if (hasSubKey(pkg, 'devDependencies', name)) {
             pkg.devDependencies[name] = newSpec
             // don't update peer or optional if we don't have to
-            if (hasSubKey(pkg, 'peerDependencies', name) && !intersects(newSpec, pkg.peerDependencies[name]))
+            if (hasSubKey(pkg, 'peerDependencies', name) && !intersects(newSpec, pkg.peerDependencies[name])) {
               pkg.peerDependencies[name] = newSpec
+            }
 
-            if (hasSubKey(pkg, 'optionalDependencies', name) && !intersects(newSpec, pkg.optionalDependencies[name]))
+            if (hasSubKey(pkg, 'optionalDependencies', name) && !intersects(newSpec, pkg.optionalDependencies[name])) {
               pkg.optionalDependencies[name] = newSpec
+            }
           } else {
-            if (hasSubKey(pkg, 'peerDependencies', name))
+            if (hasSubKey(pkg, 'peerDependencies', name)) {
               pkg.peerDependencies[name] = newSpec
+            }
 
-            if (hasSubKey(pkg, 'optionalDependencies', name))
+            if (hasSubKey(pkg, 'optionalDependencies', name)) {
               pkg.optionalDependencies[name] = newSpec
+            }
           }
         }
 
@@ -1060,8 +1263,9 @@ module.exports = cls => class Reifier extends cls {
     }
 
     // grab any from explicitRequests that had deps removed
-    for (const { from: tree } of this.explicitRequests)
+    for (const { from: tree } of this.explicitRequests) {
       updatedTrees.add(tree)
+    }
 
     for (const tree of updatedTrees) {
       // refresh the edges so they have the correct specs
@@ -1075,8 +1279,9 @@ module.exports = cls => class Reifier extends cls {
   }
 
   async [_saveLockFile] (saveOpt) {
-    if (!this[_usePackageLock])
+    if (!this[_usePackageLock]) {
       return
+    }
 
     const { meta } = this.idealTree
 
@@ -1088,8 +1293,9 @@ module.exports = cls => class Reifier extends cls {
     for (const path of this[_trashList]) {
       const loc = relpath(this.idealTree.realpath, path)
       const node = this.idealTree.inventory.get(loc)
-      if (node && node.root === this.idealTree)
+      if (node && node.root === this.idealTree) {
         node.parent = null
+      }
     }
 
     // if we filtered to only certain nodes, then anything ELSE needs
@@ -1108,54 +1314,60 @@ module.exports = cls => class Reifier extends cls {
 
         // if it's an ideal node from the filter set, then skip it
         // because we already made whatever changes were necessary
-        if (filterSet.has(ideal))
+        if (filterSet.has(ideal)) {
           continue
+        }
 
         // otherwise, if it's not in the actualTree, then it's not a thing
         // that we actually added.  And if it IS in the actualTree, then
         // it's something that we left untouched, so we need to record
         // that.
         const actual = this.actualTree.inventory.get(loc)
-        if (!actual)
+        if (!actual) {
           ideal.root = null
-        else {
+        } else {
           if ([...actual.linksIn].some(link => filterSet.has(link))) {
             seen.add(actual.location)
             continue
           }
           const { realpath, isLink } = actual
-          if (isLink && ideal.isLink && ideal.realpath === realpath)
+          if (isLink && ideal.isLink && ideal.realpath === realpath) {
             continue
-          else
+          } else {
             reroot.add(actual)
+          }
         }
       }
 
       // now find any actual nodes that may not be present in the ideal
       // tree, but were left behind by virtue of not being in the filter
       for (const [loc, actual] of this.actualTree.inventory.entries()) {
-        if (seen.has(loc))
+        if (seen.has(loc)) {
           continue
+        }
         seen.add(loc)
 
         // we know that this is something that ISN'T in the idealTree,
         // or else we will have addressed it in the previous loop.
         // If it's in the filterSet, that means we intentionally removed
         // it, so nothing to do here.
-        if (filterSet.has(actual))
+        if (filterSet.has(actual)) {
           continue
+        }
 
         reroot.add(actual)
       }
 
       // go through the rerooted actual nodes, and move them over.
-      for (const actual of reroot)
+      for (const actual of reroot) {
         actual.root = this.idealTree
+      }
 
       // prune out any tops that lack a linkIn, they are no longer relevant.
       for (const top of this.idealTree.tops) {
-        if (top.linksIn.size === 0)
+        if (top.linksIn.size === 0) {
           top.root = null
+        }
       }
 
       // need to calculate dep flags, since nodes may have been marked
@@ -1171,7 +1383,8 @@ module.exports = cls => class Reifier extends cls {
     this.actualTree = this.idealTree
     this.idealTree = null
 
-    if (!this[_global])
+    if (!this[_global]) {
       await this.actualTree.meta.save()
+    }
   }
 }