@npmcli/arborist 2.7.0 → 2.8.2

package/lib/arborist/load-actual.js

@@ -188,8 +188,10 @@ module.exports = cls => class ActualLoader extends cls {
       const tree = this[_actualTree]
       const actualRoot = tree.isLink ? tree.target : tree
       const { dependencies = {} } = actualRoot.package
-      for (const name of actualRoot.children.keys())
-        dependencies[name] = dependencies[name] || '*'
+      for (const [name, kid] of actualRoot.children.entries()) {
+        const def = kid.isLink ? `file:${kid.realpath}` : '*'
+        dependencies[name] = dependencies[name] || def
+      }
       actualRoot.package = { ...actualRoot.package, dependencies }
     }
     return this[_actualTree]

package/lib/arborist/reify.js

@@ -5,11 +5,14 @@ const pacote = require('pacote')
 const AuditReport = require('../audit-report.js')
 const {subset, intersects} = require('semver')
 const npa = require('npm-package-arg')
+const debug = require('../debug.js')
+const walkUp = require('walk-up-path')
 
 const {dirname, resolve, relative} = require('path')
 const {depth: dfwalk} = require('treeverse')
 const fs = require('fs')
 const {promisify} = require('util')
+const lstat = promisify(fs.lstat)
 const symlink = promisify(fs.symlink)
 const mkdirp = require('mkdirp-infer-owner')
 const justMkdirp = require('mkdirp')
@@ -18,6 +21,7 @@ const rimraf = promisify(require('rimraf'))
 const PackageJson = require('@npmcli/package-json')
 const packageContents = require('@npmcli/installed-package-contents')
 const { checkEngine, checkPlatform } = require('npm-install-checks')
+const _force = Symbol.for('force')
 
 const treeCheck = require('../tree-check.js')
 const relpath = require('../relpath.js')
@@ -50,6 +54,7 @@ const _createSparseTree = Symbol.for('createSparseTree')
 const _loadShrinkwrapsAndUpdateTrees = Symbol.for('loadShrinkwrapsAndUpdateTrees')
 const _shrinkwrapInflated = Symbol('shrinkwrapInflated')
 const _bundleUnpacked = Symbol('bundleUnpacked')
+const _bundleMissing = Symbol('bundleMissing')
 const _reifyNode = Symbol.for('reifyNode')
 const _extractOrLink = Symbol('extractOrLink')
 // defined by rebuild mixin
@@ -74,8 +79,10 @@ const _copyIdealToActual = Symbol('copyIdealToActual')
 const _addOmitsToTrashList = Symbol('addOmitsToTrashList')
 const _packageLockOnly = Symbol('packageLockOnly')
 const _dryRun = Symbol('dryRun')
+const _validateNodeModules = Symbol('validateNodeModules')
+const _nmValidated = Symbol('nmValidated')
 const _validatePath = Symbol('validatePath')
-const _reifyPackages = Symbol('reifyPackages')
+const _reifyPackages = Symbol.for('reifyPackages')
 
 const _omitDev = Symbol('omitDev')
 const _omitOptional = Symbol('omitOptional')
@@ -83,8 +90,9 @@ const _omitPeer = Symbol('omitPeer')
 
 const _global = Symbol.for('global')
 
+const _pruneBundledMetadeps = Symbol('pruneBundledMetadeps')
+
 // defined by Ideal mixin
-const _pruneBundledMetadeps = Symbol.for('pruneBundledMetadeps')
 const _resolvedAdd = Symbol.for('resolvedAdd')
 const _usePackageLock = Symbol.for('usePackageLock')
 const _formatPackageLock = Symbol.for('formatPackageLock')
@@ -112,6 +120,11 @@ module.exports = cls => class Reifier extends cls {
     this[_sparseTreeDirs] = new Set()
     this[_sparseTreeRoots] = new Set()
     this[_trashList] = new Set()
+    // the nodes we unpack to read their bundles
+    this[_bundleUnpacked] = new Set()
+    // child nodes we'd EXPECT to be included in a bundle, but aren't
+    this[_bundleMissing] = new Set()
+    this[_nmValidated] = new Set()
   }
 
   // public method
@@ -152,6 +165,9 @@ module.exports = cls => class Reifier extends cls {
     // recursively, because it can have other side effects to do that
     // in a project directory.  We just want to make it if it's missing.
     await justMkdirp(resolve(this.path))
+
+    // do not allow the top-level node_modules to be a symlink
+    await this[_validateNodeModules](resolve(this.path, 'node_modules'))
   }
 
   async [_reifyPackages] () {
@@ -321,12 +337,13 @@ module.exports = cls => class Reifier extends cls {
       ideal: this.idealTree,
     })
 
-    for (const node of this.diff.removed) {
-      // a node in a dep bundle will only be removed if its bundling dep
-      // is removed as well.  in which case, we don't have to delete it!
-      if (!node.inDepBundle)
-        this[_addNodeToTrashList](node)
-    }
+    // we don't have to add 'removed' folders to the trashlist, because
+    // they'll be moved aside to a retirement folder, and then the retired
+    // folder will be deleted at the end.  This is important when we have
+    // a folder like FOO being "removed" in favor of a folder like "foo",
+    // because if we remove node_modules/FOO on case-insensitive systems,
+    // it will remove the dep that we *want* at node_modules/foo.
+
     process.emit('timeEnd', 'reify:diffTrees')
   }
 
@@ -334,7 +351,7 @@ module.exports = cls => class Reifier extends cls {
   // removed later on in the process.  optionally, also mark them
   // as a retired paths, so that we move them out of the way and
   // replace them when rolling back on failure.
-  [_addNodeToTrashList] (node, retire) {
+  [_addNodeToTrashList] (node, retire = false) {
     const paths = [node.path, ...node.binPaths]
     const moves = this[_retiredPaths]
     this.log.silly('reify', 'mark', retire ? 'retired' : 'deleted', paths)
@@ -422,19 +439,40 @@ module.exports = cls => class Reifier extends cls {
     process.emit('time', 'reify:createSparse')
     // if we call this fn again, we look for the previous list
     // so that we can avoid making the same directory multiple times
-    const dirs = this.diff.leaves
+    const leaves = this.diff.leaves
       .filter(diff => {
         return (diff.action === 'ADD' || diff.action === 'CHANGE') &&
           !this[_sparseTreeDirs].has(diff.ideal.path) &&
           !diff.ideal.isLink
       })
-      .map(diff => diff.ideal.path)
-
-    return promiseAllRejectLate(dirs.map(d => mkdirp(d)))
-      .then(made => {
-        made.forEach(made => this[_sparseTreeRoots].add(made))
-        dirs.forEach(dir => this[_sparseTreeDirs].add(dir))
-      })
+      .map(diff => diff.ideal)
+
+    // we check this in parallel, so guard against multiple attempts to
+    // retire the same path at the same time.
+    const dirsChecked = new Set()
+    return promiseAllRejectLate(leaves.map(async node => {
+      for (const d of walkUp(node.path)) {
+        if (d === node.top.path)
+          break
+        if (dirsChecked.has(d))
+          continue
+        dirsChecked.add(d)
+        const st = await lstat(d).catch(er => null)
+        // this can happen if we have a link to a package with a name
+        // that the filesystem treats as if it is the same thing.
+        // would be nice to have conditional istanbul ignores here...
+        /* istanbul ignore next - defense in depth */
+        if (st && !st.isDirectory()) {
+          const retired = retirePath(d)
+          this[_retiredPaths][d] = retired
+          this[_trashList].add(retired)
+          await this[_renamePath](d, retired)
+        }
+      }
+      const made = await mkdirp(node.path)
+      this[_sparseTreeDirs].add(node.path)
+      this[_sparseTreeRoots].add(made)
+    }))
       .then(() => process.emit('timeEnd', 'reify:createSparse'))
   }
 
@@ -529,7 +567,20 @@ module.exports = cls => class Reifier extends cls {
       })
   }
 
-  [_extractOrLink] (node) {
+  // do not allow node_modules to be a symlink
+  async [_validateNodeModules] (nm) {
+    if (this[_force] || this[_nmValidated].has(nm))
+      return
+    const st = await lstat(nm).catch(() => null)
+    if (!st || st.isDirectory()) {
+      this[_nmValidated].add(nm)
+      return
+    }
+    this.log.warn('reify', 'Removing non-directory', nm)
+    await rimraf(nm)
+  }
+
+  async [_extractOrLink] (node) {
     // in normal cases, node.resolved should *always* be set by now.
     // however, it is possible when a lockfile is damaged, or very old,
     // or in some other race condition bugs in npm v6, that a previously
@@ -556,13 +607,29 @@ module.exports = cls => class Reifier extends cls {
       return
     }
 
-    return node.isLink
-      ? rimraf(node.path).then(() => this[_symlink](node))
-      : pacote.extract(res, node.path, {
+    const nm = resolve(node.parent.path, 'node_modules')
+    await this[_validateNodeModules](nm)
+
+    if (node.isLink) {
+      await rimraf(node.path)
+      await this[_symlink](node)
+    } else {
+      await debug(async () => {
+        const st = await lstat(node.path).catch(e => null)
+        if (st && !st.isDirectory()) {
+          debug.log('unpacking into a non-directory', node)
+          throw Object.assign(new Error('ENOTDIR: not a directory'), {
+            code: 'ENOTDIR',
+            path: node.path,
+          })
+        }
+      })
+      await pacote.extract(res, node.path, {
         ...this.options,
         resolved: node.resolved,
         integrity: node.integrity,
       })
+    }
   }
 
   async [_symlink] (node) {
@@ -610,10 +677,9 @@ module.exports = cls => class Reifier extends cls {
   [_loadBundlesAndUpdateTrees] (
     depth = 0, bundlesByDepth = this[_getBundlesByDepth]()
   ) {
-    if (depth === 0) {
-      this[_bundleUnpacked] = new Set()
+    if (depth === 0)
       process.emit('time', 'reify:loadBundles')
-    }
+
     const maxBundleDepth = bundlesByDepth.get('maxBundleDepth')
     if (depth > maxBundleDepth) {
       // if we did something, then prune the tree and update the diffs
@@ -642,14 +708,30 @@ module.exports = cls => class Reifier extends cls {
     }))
     // then load their unpacked children and move into the ideal tree
       .then(nodes =>
-        promiseAllRejectLate(nodes.map(node => new this.constructor({
-          ...this.options,
-          path: node.path,
-        }).loadActual({
-          root: node,
-          // don't transplant any sparse folders we created
-          transplantFilter: node => node.package._id,
-        }))))
+        promiseAllRejectLate(nodes.map(async node => {
+          const arb = new this.constructor({
+            ...this.options,
+            path: node.path,
+          })
+          const notTransplanted = new Set(node.children.keys())
+          await arb.loadActual({
+            root: node,
+            // don't transplant any sparse folders we created
+            // loadActual will set node.package to {} for empty directories
+            // if by chance there are some empty folders in the node_modules
+            // tree for some other reason, then ok, ignore those too.
+            transplantFilter: node => {
+              if (node.package._id) {
+                // it's actually in the bundle if it gets transplanted
+                notTransplanted.delete(node.name)
+                return true
+              } else
+                return false
+            },
+          })
+          for (const name of notTransplanted)
+            this[_bundleMissing].add(node.children.get(name))
+        })))
     // move onto the next level of bundled items
       .then(() => this[_loadBundlesAndUpdateTrees](depth + 1, bundlesByDepth))
   }
@@ -685,6 +767,27 @@ module.exports = cls => class Reifier extends cls {
   // https://github.com/npm/cli/issues/1597#issuecomment-667639545
   [_pruneBundledMetadeps] (bundlesByDepth) {
     const bundleShadowed = new Set()
+
+    // Example dep graph:
+    // root -> (a, c)
+    // a -> BUNDLE(b)
+    // b -> c
+    // c -> b
+    //
+    // package tree:
+    // root
+    // +-- a
+    // |   +-- b(1)
+    // |   +-- c(1)
+    // +-- b(2)
+    // +-- c(2)
+    // 1. mark everything that's shadowed by anything in the bundle.  This
+    //    marks b(2) and c(2).
+    // 2. anything with edgesIn from outside the set, mark not-extraneous,
+    //    remove from set.  This unmarks c(2).
+    // 3. continue until no change
+    // 4. remove everything in the set from the tree.  b(2) is pruned
+
     // create the list of nodes shadowed by children of bundlers
     for (const bundles of bundlesByDepth.values()) {
       // skip the 'maxBundleDepth' item
@@ -700,36 +803,50 @@ module.exports = cls => class Reifier extends cls {
         }
       }
     }
-    let changed = true
-    while (changed) {
-      changed = false
-      for (const shadow of bundleShadowed) {
-        if (!shadow.extraneous) {
-          bundleShadowed.delete(shadow)
-          continue
+
+    // lib -> (a@1.x) BUNDLE(a@1.2.3 (b@1.2.3))
+    // a@1.2.3 -> (b@1.2.3)
+    // a@1.3.0 -> (b@2)
+    // b@1.2.3 -> ()
+    // b@2 -> (c@2)
+    //
+    // root
+    // +-- lib
+    // |   +-- a@1.2.3
+    // |   +-- b@1.2.3
+    // +-- b@2 <-- shadowed, now extraneous
+    // +-- c@2 <-- also shadowed, because only dependent is shadowed
+    for (const shadow of bundleShadowed) {
+      for (const shadDep of shadow.edgesOut.values()) {
+        /* istanbul ignore else - pretty unusual situation, just being
+         * defensive here. Would mean that a bundled dep has a dependency
+         * that is unmet. which, weird, but if you bundle it, we take
+         * whatever you put there and assume the publisher knows best. */
+        if (shadDep.to) {
+          bundleShadowed.add(shadDep.to)
+          shadDep.to.extraneous = true
         }
+      }
+    }
 
+    let changed
+    do {
+      changed = false
+      for (const shadow of bundleShadowed) {
         for (const edge of shadow.edgesIn) {
-          if (!edge.from.extraneous) {
+          if (!bundleShadowed.has(edge.from)) {
             shadow.extraneous = false
             bundleShadowed.delete(shadow)
             changed = true
-          } else {
-            for (const shadDep of shadow.edgesOut.values()) {
-              /* istanbul ignore else - pretty unusual situation, just being
-               * defensive here. Would mean that a bundled dep has a dependency
-               * that is unmet. which, weird, but if you bundle it, we take
-               * whatever you put there and assume the publisher knows best. */
-              if (shadDep.to)
-                bundleShadowed.add(shadDep.to)
-            }
+            break
           }
         }
       }
-    }
+    } while (changed)
+
     for (const shadow of bundleShadowed) {
-      shadow.parent = null
       this[_addNodeToTrashList](shadow)
+      shadow.root = null
     }
   }
 
@@ -780,6 +897,7 @@ module.exports = cls => class Reifier extends cls {
         const node = diff.ideal
         const bd = this[_bundleUnpacked].has(node)
         const sw = this[_shrinkwrapInflated].has(node)
+        const bundleMissing = this[_bundleMissing].has(node)
 
         // check whether we still need to unpack this one.
         // test the inDepBundle last, since that's potentially a tree walk.
@@ -787,7 +905,7 @@ module.exports = cls => class Reifier extends cls {
           !node.isRoot && // root node already exists
           !bd && // already unpacked to read bundle
           !sw && // already unpacked to read sw
-          !node.inDepBundle // already unpacked by another dep's bundle
+          (bundleMissing || !node.inDepBundle) // already unpacked by another dep's bundle
 
         if (doUnpack)
           unpacks.push(this[_reifyNode](node))
@@ -814,8 +932,26 @@ module.exports = cls => class Reifier extends cls {
     const moves = this[_retiredPaths]
     this[_retiredUnchanged] = {}
     return promiseAllRejectLate(this.diff.children.map(diff => {
-      const realFolder = (diff.actual || diff.ideal).path
+      // skip if nothing was retired
+      if (diff.action !== 'CHANGE' && diff.action !== 'REMOVE')
+        return
+
+      const { path: realFolder } = diff.actual
       const retireFolder = moves[realFolder]
+      /* istanbul ignore next - should be impossible */
+      debug(() => {
+        if (!retireFolder) {
+          const er = new Error('trying to un-retire but not retired')
+          throw Object.assign(er, {
+            realFolder,
+            retireFolder,
+            actual: diff.actual,
+            ideal: diff.ideal,
+            action: diff.action,
+          })
+        }
+      })
+
       this[_retiredUnchanged][retireFolder] = []
       return promiseAllRejectLate(diff.unchanged.map(node => {
         // no need to roll back links, since we'll just delete them anyway
@@ -823,7 +959,7 @@ module.exports = cls => class Reifier extends cls {
           return mkdirp(dirname(node.path)).then(() => this[_reifyNode](node))
 
         // will have been moved/unpacked along with bundler
-        if (node.inDepBundle)
+        if (node.inDepBundle && !this[_bundleMissing].has(node))
           return
 
         this[_retiredUnchanged][retireFolder].push(node)