@npmcli/arborist 2.3.0 → 2.4.0

package/bin/lib/logging.js

@@ -20,13 +20,21 @@ const levelMap = new Map(levels.reduce((set, level, index) => {
 }, []))
 
 const { inspect, format } = require('util')
+const colors = process.stderr.isTTY
+const magenta = colors ? msg => `\x1B[35m${msg}\x1B[39m` : m => m
 if (loglevel !== 'silent') {
   process.on('log', (level, ...args) => {
     if (levelMap.get(level) < levelMap.get(loglevel))
       return
-    const pref = `${process.pid} ${level} `
+    const pref = `${process.pid} ${magenta(level)} `
     if (level === 'warn' && args[0] === 'ERESOLVE')
-      args[2] = inspect(args[2], { depth: 10 })
+      args[2] = inspect(args[2], { depth: 10, colors })
+    else {
+      args = args.map(a => {
+        return typeof a === 'string' ? a
+          : inspect(a, { depth: 10, colors })
+      })
+    }
     const msg = pref + format(...args).trim().split('\n').join(`\n${pref}`)
     console.error(msg)
   })

package/bin/lib/timers.js

@@ -1,4 +1,5 @@
 const timers = Object.create(null)
+const { format } = require('util')
 
 process.on('time', name => {
   if (timers[name])
@@ -6,17 +7,20 @@ process.on('time', name => {
   timers[name] = process.hrtime()
 })
 
+const dim = process.stderr.isTTY ? msg => `\x1B[2m${msg}\x1B[22m` : m => m
+const red = process.stderr.isTTY ? msg => `\x1B[31m${msg}\x1B[39m` : m => m
 process.on('timeEnd', name => {
   if (!timers[name])
     throw new Error('timer not started! ' + name)
   const res = process.hrtime(timers[name])
   delete timers[name]
-  console.error(`${process.pid} ${name}`, res[0] * 1e3 + res[1] / 1e6)
+  const msg = format(`${process.pid} ${name}`, res[0] * 1e3 + res[1] / 1e6)
+  console.error(dim(msg))
 })
 
 process.on('exit', () => {
   for (const name of Object.keys(timers)) {
-    console.error('Dangling timer: ', name)
+    console.error(red('Dangling timer:'), name)
     process.exitCode = 1
   }
 })

package/bin/virtual.js

@@ -8,7 +8,8 @@ require('./lib/timers.js')
 const start = process.hrtime()
 new Arborist(options).loadVirtual().then(tree => {
   const end = process.hrtime(start)
-  print(tree)
+  if (!options.quiet)
+    print(tree)
   if (options.save)
     tree.meta.save()
   console.error(`read ${tree.inventory.size} deps in ${end[0] * 1000 + end[1] / 1e6}ms`)

package/lib/arborist/build-ideal-tree.js

@@ -1188,6 +1188,16 @@ This is a one-time fix-up, please be patient...
         }
       }
 
+      // There is something present already, and we're not happy about it
+      // See if the thing we WOULD be happy with is also going to satisfy
+      // the other dependents on the current node.
+      const current = edge.to
+      const dep = await this[_nodeFromEdge](edge, null, null, required)
+      if (dep.canReplace(current)) {
+        await this[_nodeFromEdge](edge, node.parent, null, required)
+        continue
+      }
+
       // at this point we know that there is a dep there, and
       // we don't like it.  always fail strictly, always allow forcibly or
       // in non-strict mode if it's not our fault.  don't warn here, because

package/lib/arborist/index.js

@@ -54,7 +54,7 @@ class Arborist extends Base {
       ...options,
       path: options.path || '.',
       cache: options.cache || `${homedir()}/.npm/_cacache`,
-      packumentCache: new Map(),
+      packumentCache: options.packumentCache || new Map(),
       log: options.log || procLog,
     }
     this.cache = resolve(this.options.cache)

package/lib/arborist/load-virtual.js

@@ -93,7 +93,8 @@ module.exports = cls => class VirtualLoader extends cls {
     this.virtualTree = root
     const {links, nodes} = this[resolveNodes](s, root)
     await this[resolveLinks](links, nodes)
-    this[assignBundles](nodes)
+    if (!(s.originalLockfileVersion >= 2))
+      this[assignBundles](nodes)
     if (this[flagsSuspect])
       this[reCalcDepFlags](nodes.values())
     return root
@@ -220,22 +221,24 @@ module.exports = cls => class VirtualLoader extends cls {
   [assignBundles] (nodes) {
     for (const [location, node] of nodes) {
       // Skip assignment of parentage for the root package
-      if (!location)
+      if (!location || node.target && !node.target.location)
         continue
       const { name, parent, package: { inBundle }} = node
+
       if (!parent)
         continue
 
       // read inBundle from package because 'package' here is
       // actually a v2 lockfile metadata entry.
-      // If the *parent* is also bundled, though, then we assume
-      // that it's being pulled in just by virtue of that.
+      // If the *parent* is also bundled, though, or if the parent has
+      // no dependency on it, then we assume that it's being pulled in
+      // just by virtue of its parent or a transitive dep being bundled.
       const { package: ppkg } = parent
       const { inBundle: parentBundled } = ppkg
-      if (inBundle && !parentBundled) {
+      if (inBundle && !parentBundled && parent.edgesOut.has(node.name)) {
         if (!ppkg.bundleDependencies)
           ppkg.bundleDependencies = [name]
-        else if (!ppkg.bundleDependencies.includes(name))
+        else
           ppkg.bundleDependencies.push(name)
       }
     }

package/lib/arborist/rebuild.js

@@ -115,10 +115,6 @@ module.exports = cls => class Builder extends cls {
       await this[_runScripts]('preinstall')
     if (this[_binLinks] && type !== 'links')
       await this[_linkAllBins]()
-    if (!this[_ignoreScripts]) {
-      await this[_runScripts]('install')
-      await this[_runScripts]('postinstall')
-    }
 
     // links should also run prepare scripts and only link bins after that
     if (type === 'links') {
@@ -128,6 +124,11 @@ module.exports = cls => class Builder extends cls {
         await this[_linkAllBins]()
     }
 
+    if (!this[_ignoreScripts]) {
+      await this[_runScripts]('install')
+      await this[_runScripts]('postinstall')
+    }
+
     process.emit('timeEnd', `build:${type}`)
   }
 

package/lib/arborist/reify.js

@@ -49,7 +49,8 @@ const _loadTrees = Symbol.for('loadTrees')
 const _diffTrees = Symbol.for('diffTrees')
 const _createSparseTree = Symbol.for('createSparseTree')
 const _loadShrinkwrapsAndUpdateTrees = Symbol.for('loadShrinkwrapsAndUpdateTrees')
-const _shrinkwrapUnpacked = Symbol('shrinkwrapUnpacked')
+const _shrinkwrapInflated = Symbol('shrinkwrapInflated')
+const _bundleUnpacked = Symbol('bundleUnpacked')
 const _reifyNode = Symbol.for('reifyNode')
 const _extractOrLink = Symbol('extractOrLink')
 // defined by rebuild mixin
@@ -108,7 +109,7 @@ module.exports = cls => class Reifier extends cls {
 
     this.diff = null
     this[_retiredPaths] = {}
-    this[_shrinkwrapUnpacked] = new Set()
+    this[_shrinkwrapInflated] = new Set()
     this[_retiredUnchanged] = {}
     this[_sparseTreeDirs] = new Set()
     this[_sparseTreeRoots] = new Set()
@@ -316,6 +317,7 @@ module.exports = cls => class Reifier extends cls {
     // find all the nodes that need to change between the actual
     // and ideal trees.
     this.diff = Diff.calculate({
+      shrinkwrapInflated: this[_shrinkwrapInflated],
       filterNodes,
       actual: this.actualTree,
       ideal: this.idealTree,
@@ -423,7 +425,8 @@ module.exports = cls => class Reifier extends cls {
     const dirs = this.diff.leaves
       .filter(diff => {
         return (diff.action === 'ADD' || diff.action === 'CHANGE') &&
-          !this[_sparseTreeDirs].has(diff.ideal.path)
+          !this[_sparseTreeDirs].has(diff.ideal.path) &&
+          !diff.ideal.isLink
       })
       .map(diff => diff.ideal.path)
 
@@ -457,9 +460,9 @@ module.exports = cls => class Reifier extends cls {
   // we need to unpack them, read that shrinkwrap file, and then update
   // the tree by calling loadVirtual with the node as the root.
   [_loadShrinkwrapsAndUpdateTrees] () {
-    const seen = this[_shrinkwrapUnpacked]
+    const seen = this[_shrinkwrapInflated]
     const shrinkwraps = this.diff.leaves
-      .filter(d => (d.action === 'CHANGE' || d.action === 'ADD') &&
+      .filter(d => (d.action === 'CHANGE' || d.action === 'ADD' || !d.action) &&
         d.ideal.hasShrinkwrap && !seen.has(d.ideal) &&
         !this[_trashList].has(d.ideal.path))
 
@@ -472,7 +475,7 @@ module.exports = cls => class Reifier extends cls {
     return promiseAllRejectLate(shrinkwraps.map(diff => {
       const node = diff.ideal
       seen.add(node)
-      return this[_reifyNode](node)
+      return diff.action ? this[_reifyNode](node) : node
     }))
       .then(nodes => promiseAllRejectLate(nodes.map(node => new Arborist({
         ...this.options,
@@ -503,7 +506,7 @@ module.exports = cls => class Reifier extends cls {
 
     const { npmVersion, nodeVersion } = this.options
     const p = Promise.resolve()
-      .then(() => {
+      .then(async () => {
         // when we reify an optional node, check the engine and platform
         // first. be sure to ignore the --force and --engine-strict flags,
         // since we always want to skip any optional packages we can't install.
@@ -513,11 +516,11 @@ module.exports = cls => class Reifier extends cls {
           checkEngine(node.package, npmVersion, nodeVersion, false)
           checkPlatform(node.package, false)
         }
+        await this[_checkBins](node)
+        await this[_extractOrLink](node)
+        await this[_warnDeprecated](node)
+        await this[_loadAncientPackageDetails](node)
       })
-      .then(() => this[_checkBins](node))
-      .then(() => this[_extractOrLink](node))
-      .then(() => this[_warnDeprecated](node))
-      .then(() => this[_loadAncientPackageDetails](node))
 
     return this[_handleOptionalFailure](node, p)
       .then(() => {
@@ -563,10 +566,11 @@ module.exports = cls => class Reifier extends cls {
       })
   }
 
-  [_symlink] (node) {
+  async [_symlink] (node) {
     const dir = dirname(node.path)
     const target = node.realpath
     const rel = relative(dir, target)
+    await mkdirp(dir)
     return symlink(rel, node.path, 'junction')
   }
 
@@ -633,8 +637,10 @@ module.exports = cls => class Reifier extends cls {
   [_loadBundlesAndUpdateTrees] (
     depth = 0, bundlesByDepth = this[_getBundlesByDepth]()
   ) {
-    if (depth === 0)
+    if (depth === 0) {
+      this[_bundleUnpacked] = new Set()
       process.emit('time', 'reify:loadBundles')
+    }
     const maxBundleDepth = bundlesByDepth.get('maxBundleDepth')
     if (depth > maxBundleDepth) {
       // if we did something, then prune the tree and update the diffs
@@ -650,13 +656,17 @@ module.exports = cls => class Reifier extends cls {
     // shallower bundle overwriting them with a bundled meta-dep.
     const set = (bundlesByDepth.get(depth) || [])
       .filter(node => node.root === this.idealTree &&
+        node.target !== node.root &&
         !this[_trashList].has(node.path))
 
     if (!set.length)
       return this[_loadBundlesAndUpdateTrees](depth + 1, bundlesByDepth)
 
     // extract all the nodes with bundles
-    return promiseAllRejectLate(set.map(node => this[_reifyNode](node)))
+    return promiseAllRejectLate(set.map(node => {
+      this[_bundleUnpacked].add(node)
+      return this[_reifyNode](node)
+    }))
     // then load their unpacked children and move into the ideal tree
       .then(nodes =>
         promiseAllRejectLate(nodes.map(node => new this.constructor({
@@ -678,8 +688,13 @@ module.exports = cls => class Reifier extends cls {
       tree: this.diff,
       visit: diff => {
         const node = diff.ideal
-        if (node && !node.isProjectRoot && node.package.bundleDependencies &&
-            node.package.bundleDependencies.length) {
+        if (!node)
+          return
+        if (node.isProjectRoot || (node.target && node.target.isProjectRoot))
+          return
+
+        const { bundleDependencies } = node.package
+        if (bundleDependencies && bundleDependencies.length) {
           maxBundleDepth = Math.max(maxBundleDepth, node.depth)
           if (!bundlesByDepth.has(node.depth))
             bundlesByDepth.set(node.depth, [node])
@@ -784,14 +799,14 @@ module.exports = cls => class Reifier extends cls {
           return
 
         const node = diff.ideal
-        const bd = node.package.bundleDependencies
-        const sw = this[_shrinkwrapUnpacked].has(node)
+        const bd = this[_bundleUnpacked].has(node)
+        const sw = this[_shrinkwrapInflated].has(node)
 
         // check whether we still need to unpack this one.
         // test the inDepBundle last, since that's potentially a tree walk.
         const doUnpack = node && // can't unpack if removed!
           !node.isRoot && // root node already exists
-          !(bd && bd.length) && // already unpacked to read bundle
+          !bd && // already unpacked to read bundle
           !sw && // already unpacked to read sw
           !node.inDepBundle // already unpacked by another dep's bundle
 

package/lib/debug.js

@@ -12,6 +12,7 @@
 
 // run in debug mode if explicitly requested, running arborist tests,
 // or working in the arborist project directory.
+
 const debug = process.env.ARBORIST_DEBUG !== '0' && (
   process.env.ARBORIST_DEBUG === '1' ||
   /\barborist\b/.test(process.env.NODE_DEBUG || '') ||
@@ -21,4 +22,10 @@ const debug = process.env.ARBORIST_DEBUG !== '0' && (
 )
 
 module.exports = debug ? fn => fn() : () => {}
-module.exports.log = (...msg) => module.exports(() => console.error(...msg))
+const red = process.stderr.isTTY ? msg => `\x1B[31m${msg}\x1B[39m` : m => m
+module.exports.log = (...msg) => module.exports(() => {
+  const { format } = require('util')
+  const prefix = `\n${process.pid} ${red(format(msg.shift()))} `
+  msg = (prefix + format(...msg).trim().split('\n').join(prefix)).trim()
+  console.error(msg)
+})

package/lib/diff.js

@@ -11,8 +11,9 @@ const {existsSync} = require('fs')
 const ssri = require('ssri')
 
 class Diff {
-  constructor ({actual, ideal, filterSet}) {
+  constructor ({actual, ideal, filterSet, shrinkwrapInflated}) {
     this.filterSet = filterSet
+    this.shrinkwrapInflated = shrinkwrapInflated
     this.children = []
     this.actual = actual
     this.ideal = ideal
@@ -30,7 +31,7 @@ class Diff {
     this.removed = []
   }
 
-  static calculate ({actual, ideal, filterNodes = []}) {
+  static calculate ({actual, ideal, filterNodes = [], shrinkwrapInflated = new Set()}) {
     // if there's a filterNode, then:
     // - get the path from the root to the filterNode.  The root or
     //   root.target should have an edge either to the filterNode or
@@ -77,7 +78,7 @@ class Diff {
     }
 
     return depth({
-      tree: new Diff({actual, ideal, filterSet}),
+      tree: new Diff({actual, ideal, filterSet, shrinkwrapInflated}),
       getChildren,
       leave,
     })
@@ -135,7 +136,7 @@ const allChildren = node => {
 // to create the diff tree
 const getChildren = diff => {
   const children = []
-  const {actual, ideal, unchanged, removed, filterSet} = diff
+  const {actual, ideal, unchanged, removed, filterSet, shrinkwrapInflated} = diff
 
   // Note: we DON'T diff fsChildren themselves, because they are either
   // included in the package contents, or part of some other project, and
@@ -144,11 +145,20 @@ const getChildren = diff => {
   // responsible for installing.
   const actualKids = allChildren(actual)
   const idealKids = allChildren(ideal)
+
+  if (ideal && ideal.hasShrinkwrap && !shrinkwrapInflated.has(ideal)) {
+    // Guaranteed to get a diff.leaves here, because we always
+    // be called with a proper Diff object when ideal has a shrinkwrap
+    // that has not been inflated.
+    diff.leaves.push(diff)
+    return children
+  }
+
   const paths = new Set([...actualKids.keys(), ...idealKids.keys()])
   for (const path of paths) {
     const actual = actualKids.get(path)
     const ideal = idealKids.get(path)
-    diffNode(actual, ideal, children, unchanged, removed, filterSet)
+    diffNode(actual, ideal, children, unchanged, removed, filterSet, shrinkwrapInflated)
   }
 
   if (diff.leaves && !children.length)
@@ -157,7 +167,7 @@ const getChildren = diff => {
   return children
 }
 
-const diffNode = (actual, ideal, children, unchanged, removed, filterSet) => {
+const diffNode = (actual, ideal, children, unchanged, removed, filterSet, shrinkwrapInflated) => {
   if (filterSet.size && !(filterSet.has(ideal) || filterSet.has(actual)))
     return
 
@@ -165,10 +175,10 @@ const diffNode = (actual, ideal, children, unchanged, removed, filterSet) => {
 
   // if it's a match, then get its children
   // otherwise, this is the child diff node
-  if (action) {
+  if (action || (!shrinkwrapInflated.has(ideal) && ideal.hasShrinkwrap)) {
     if (action === 'REMOVE')
       removed.push(actual)
-    children.push(new Diff({actual, ideal, filterSet}))
+    children.push(new Diff({actual, ideal, filterSet, shrinkwrapInflated}))
   } else {
     unchanged.push(ideal)
     // !*! Weird dirty hack warning !*!
@@ -199,7 +209,7 @@ const diffNode = (actual, ideal, children, unchanged, removed, filterSet) => {
       for (const node of bundledChildren)
         node.parent = ideal
     }
-    children.push(...getChildren({actual, ideal, unchanged, removed, filterSet}))
+    children.push(...getChildren({actual, ideal, unchanged, removed, filterSet, shrinkwrapInflated}))
   }
 }
 

package/lib/index.js

@@ -3,5 +3,6 @@ module.exports.Arborist = module.exports
 module.exports.Node = require('./node.js')
 module.exports.Link = require('./link.js')
 module.exports.Edge = require('./edge.js')
+module.exports.Shrinkwrap = require('./shrinkwrap.js')
 // XXX export the other classes, too.  shrinkwrap, diff, etc.
 // they're handy!

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@npmcli/arborist",
-  "version": "2.3.0",
+  "version": "2.4.0",
   "description": "Manage node_modules trees",
   "dependencies": {
     "@npmcli/installed-package-contents": "^1.0.7",
@@ -19,7 +19,7 @@
     "npm-install-checks": "^4.0.0",
     "npm-package-arg": "^8.1.0",
     "npm-pick-manifest": "^6.1.0",
-    "npm-registry-fetch": "^9.0.0",
+    "npm-registry-fetch": "^10.0.0",
     "pacote": "^11.2.6",
     "parse-conflict-json": "^1.1.1",
     "promise-all-reject-late": "^1.0.0",
@@ -41,8 +41,7 @@
     "eslint-plugin-standard": "^4.0.1",
     "minify-registry-metadata": "^2.1.0",
     "mutate-fs": "^2.1.1",
-    "require-inject": "^1.4.4",
-    "tap": "^14.11.0",
+    "tap": "^15.0.4",
     "tcompare": "^3.0.4"
   },
   "scripts": {
@@ -76,10 +75,8 @@
     "arborist": "bin/index.js"
   },
   "tap": {
-    "100": true,
     "after": "test/fixtures/cleanup.js",
     "coverage-map": "map.js",
-    "esm": false,
     "test-env": [
       "NODE_OPTIONS=--no-warnings"
     ],
@@ -87,6 +84,6 @@
       "--no-warnings",
       "--no-deprecation"
     ],
-    "timeout": "120"
+    "timeout": "240"
   }
 }