@npmcli/arborist 7.3.0 → 7.4.0

package/lib/arborist/rebuild.js

@@ -7,7 +7,7 @@ const promiseAllRejectLate = require('promise-all-reject-late')
 const rpj = require('read-package-json-fast')
 const binLinks = require('bin-links')
 const runScript = require('@npmcli/run-script')
-const promiseCallLimit = require('promise-call-limit')
+const { callLimit: promiseCallLimit } = require('promise-call-limit')
 const { resolve } = require('path')
 const {
   isNodeGypPackage,
@@ -19,67 +19,37 @@ const boolEnv = b => b ? '1' : ''
 const sortNodes = (a, b) =>
   (a.depth - b.depth) || localeCompare(a.path, b.path)
 
-const _workspaces = Symbol.for('workspaces')
-const _build = Symbol('build')
-const _loadDefaultNodes = Symbol('loadDefaultNodes')
-const _retrieveNodesByType = Symbol('retrieveNodesByType')
-const _resetQueues = Symbol('resetQueues')
-const _rebuildBundle = Symbol('rebuildBundle')
-const _ignoreScripts = Symbol('ignoreScripts')
-const _binLinks = Symbol('binLinks')
-const _oldMeta = Symbol('oldMeta')
-const _createBinLinks = Symbol('createBinLinks')
-const _doHandleOptionalFailure = Symbol('doHandleOptionalFailure')
-const _linkAllBins = Symbol('linkAllBins')
-const _runScripts = Symbol('runScripts')
-const _buildQueues = Symbol('buildQueues')
-const _addToBuildSet = Symbol('addToBuildSet')
 const _checkBins = Symbol.for('checkBins')
-const _queues = Symbol('queues')
-const _scriptShell = Symbol('scriptShell')
-const _includeWorkspaceRoot = Symbol.for('includeWorkspaceRoot')
-const _workspacesEnabled = Symbol.for('workspacesEnabled')
-
-const _force = Symbol.for('force')
-const _global = Symbol.for('global')
 
 // defined by reify mixin
 const _handleOptionalFailure = Symbol.for('handleOptionalFailure')
 const _trashList = Symbol.for('trashList')
 
 module.exports = cls => class Builder extends cls {
+  #doHandleOptionalFailure
+  #oldMeta = null
+  #queues
+
   constructor (options) {
     super(options)
 
-    const {
-      ignoreScripts = false,
-      scriptShell,
-      binLinks = true,
-      rebuildBundle = true,
-    } = options
-
     this.scriptsRun = new Set()
-    this[_binLinks] = binLinks
-    this[_ignoreScripts] = !!ignoreScripts
-    this[_scriptShell] = scriptShell
-    this[_rebuildBundle] = !!rebuildBundle
-    this[_resetQueues]()
-    this[_oldMeta] = null
+    this.#resetQueues()
   }
 
   async rebuild ({ nodes, handleOptionalFailure = false } = {}) {
     // nothing to do if we're not building anything!
-    if (this[_ignoreScripts] && !this[_binLinks]) {
+    if (this.options.ignoreScripts && !this.options.binLinks) {
       return
     }
 
     // when building for the first time, as part of reify, we ignore
     // failures in optional nodes, and just delete them.  however, when
     // running JUST a rebuild, we treat optional failures as real fails
-    this[_doHandleOptionalFailure] = handleOptionalFailure
+    this.#doHandleOptionalFailure = handleOptionalFailure
 
     if (!nodes) {
-      nodes = await this[_loadDefaultNodes]()
+      nodes = await this.#loadDefaultNodes()
     }
 
     // separates links nodes so that it can run
@@ -89,15 +59,15 @@ module.exports = cls => class Builder extends cls {
     const {
       depNodes,
       linkNodes,
-    } = this[_retrieveNodesByType](nodes)
+    } = this.#retrieveNodesByType(nodes)
 
     // build regular deps
-    await this[_build](depNodes, {})
+    await this.#build(depNodes, {})
 
     // build link deps
     if (linkNodes.size) {
-      this[_resetQueues]()
-      await this[_build](linkNodes, { type: 'links' })
+      this.#resetQueues()
+      await this.#build(linkNodes, { type: 'links' })
     }
 
     process.emit('timeEnd', 'build')
@@ -105,20 +75,20 @@ module.exports = cls => class Builder extends cls {
 
   // if we don't have a set of nodes, then just rebuild
   // the actual tree on disk.
-  async [_loadDefaultNodes] () {
+  async #loadDefaultNodes () {
     let nodes
     const tree = await this.loadActual()
     let filterSet
-    if (!this[_workspacesEnabled]) {
+    if (!this.options.workspacesEnabled) {
       filterSet = this.excludeWorkspacesDependencySet(tree)
       nodes = tree.inventory.filter(node =>
         filterSet.has(node) || node.isProjectRoot
       )
-    } else if (this[_workspaces] && this[_workspaces].length) {
+    } else if (this.options.workspaces.length) {
       filterSet = this.workspaceDependencySet(
         tree,
-        this[_workspaces],
-        this[_includeWorkspaceRoot]
+        this.options.workspaces,
+        this.options.includeWorkspaceRoot
       )
       nodes = tree.inventory.filter(node => filterSet.has(node))
     } else {
@@ -127,7 +97,7 @@ module.exports = cls => class Builder extends cls {
     return nodes
   }
 
-  [_retrieveNodesByType] (nodes) {
+  #retrieveNodesByType (nodes) {
     const depNodes = new Set()
     const linkNodes = new Set()
     const storeNodes = new Set()
@@ -154,7 +124,7 @@ module.exports = cls => class Builder extends cls {
     //
     // we avoid doing so if global=true since `bin-links` relies
     // on having the target nodes available in global mode.
-    if (!this[_global]) {
+    if (!this.options.global) {
       for (const node of linkNodes) {
         depNodes.delete(node.target)
       }
@@ -166,8 +136,8 @@ module.exports = cls => class Builder extends cls {
     }
   }
 
-  [_resetQueues] () {
-    this[_queues] = {
+  #resetQueues () {
+    this.#queues = {
       preinstall: [],
       install: [],
       postinstall: [],
@@ -176,46 +146,46 @@ module.exports = cls => class Builder extends cls {
     }
   }
 
-  async [_build] (nodes, { type = 'deps' }) {
+  async #build (nodes, { type = 'deps' }) {
     process.emit('time', `build:${type}`)
 
-    await this[_buildQueues](nodes)
+    await this.#buildQueues(nodes)
 
-    if (!this[_ignoreScripts]) {
-      await this[_runScripts]('preinstall')
+    if (!this.options.ignoreScripts) {
+      await this.#runScripts('preinstall')
     }
 
     // links should run prepare scripts and only link bins after that
     if (type === 'links') {
-      await this[_runScripts]('prepare')
+      await this.#runScripts('prepare')
     }
-    if (this[_binLinks]) {
-      await this[_linkAllBins]()
+    if (this.options.binLinks) {
+      await this.#linkAllBins()
     }
 
-    if (!this[_ignoreScripts]) {
-      await this[_runScripts]('install')
-      await this[_runScripts]('postinstall')
+    if (!this.options.ignoreScripts) {
+      await this.#runScripts('install')
+      await this.#runScripts('postinstall')
     }
 
     process.emit('timeEnd', `build:${type}`)
   }
 
-  async [_buildQueues] (nodes) {
+  async #buildQueues (nodes) {
     process.emit('time', 'build:queue')
     const set = new Set()
 
     const promises = []
     for (const node of nodes) {
-      promises.push(this[_addToBuildSet](node, set))
+      promises.push(this.#addToBuildSet(node, set))
 
       // if it has bundle deps, add those too, if rebuildBundle
-      if (this[_rebuildBundle] !== false) {
+      if (this.options.rebuildBundle !== false) {
         const bd = node.package.bundleDependencies
         if (bd && bd.length) {
           dfwalk({
             tree: node,
-            leave: node => promises.push(this[_addToBuildSet](node, set)),
+            leave: node => promises.push(this.#addToBuildSet(node, set)),
             getChildren: node => [...node.children.values()],
             filter: node => node.inBundle,
           })
@@ -236,7 +206,7 @@ module.exports = cls => class Builder extends cls {
       const tests = { bin, preinstall, install, postinstall, prepare }
       for (const [key, has] of Object.entries(tests)) {
         if (has) {
-          this[_queues][key].push(node)
+          this.#queues[key].push(node)
         }
       }
     }
@@ -249,21 +219,21 @@ module.exports = cls => class Builder extends cls {
     // the node path.  Otherwise a package can have a preinstall script
     // that unlinks something, to allow them to silently overwrite system
     // binaries, which is unsafe and insecure.
-    if (!node.globalTop || this[_force]) {
+    if (!node.globalTop || this.options.force) {
       return
     }
     const { path, package: pkg } = node
     await binLinks.checkBins({ pkg, path, top: true, global: true })
   }
 
-  async [_addToBuildSet] (node, set, refreshed = false) {
+  async #addToBuildSet (node, set, refreshed = false) {
     if (set.has(node)) {
       return
     }
 
-    if (this[_oldMeta] === null) {
+    if (this.#oldMeta === null) {
       const { root: { meta } } = node
-      this[_oldMeta] = meta && meta.loadedFromDisk &&
+      this.#oldMeta = meta && meta.loadedFromDisk &&
         !(meta.originalLockfileVersion >= 2)
     }
 
@@ -272,7 +242,7 @@ module.exports = cls => class Builder extends cls {
 
     const { preinstall, install, postinstall, prepare } = scripts
     const anyScript = preinstall || install || postinstall || prepare
-    if (!refreshed && !anyScript && (hasInstallScript || this[_oldMeta])) {
+    if (!refreshed && !anyScript && (hasInstallScript || this.#oldMeta)) {
       // we either have an old metadata (and thus might have scripts)
       // or we have an indication that there's install scripts (but
       // don't yet know what they are) so we have to load the package.json
@@ -286,7 +256,7 @@ module.exports = cls => class Builder extends cls {
 
       const { scripts = {} } = pkg
       node.package.scripts = scripts
-      return this[_addToBuildSet](node, set, true)
+      return this.#addToBuildSet(node, set, true)
     }
 
     // Rebuild node-gyp dependencies lacking an install or preinstall script
@@ -309,8 +279,8 @@ module.exports = cls => class Builder extends cls {
     }
   }
 
-  async [_runScripts] (event) {
-    const queue = this[_queues][event]
+  async #runScripts (event) {
+    const queue = this.#queues[event]
 
     if (!queue.length) {
       return
@@ -358,7 +328,7 @@ module.exports = cls => class Builder extends cls {
         pkg,
         stdio,
         env,
-        scriptShell: this[_scriptShell],
+        scriptShell: this.options.scriptShell,
       }
       const p = runScript(runOpts).catch(er => {
         const { code, signal } = er
@@ -382,17 +352,17 @@ module.exports = cls => class Builder extends cls {
         log.info('run', pkg._id, event, { code, signal })
       })
 
-      await (this[_doHandleOptionalFailure]
+      await (this.#doHandleOptionalFailure
         ? this[_handleOptionalFailure](node, p)
         : p)
 
       process.emit('timeEnd', timer)
-    }), limit)
+    }), { limit })
     process.emit('timeEnd', `build:run:${event}`)
   }
 
-  async [_linkAllBins] () {
-    const queue = this[_queues].bin
+  async #linkAllBins () {
+    const queue = this.#queues.bin
     if (!queue.length) {
       return
     }
@@ -402,14 +372,15 @@ module.exports = cls => class Builder extends cls {
     // sort the queue by node path, so that the module-local collision
     // detector in bin-links will always resolve the same way.
     for (const node of queue.sort(sortNodes)) {
-      promises.push(this[_createBinLinks](node))
+      // TODO these run before they're awaited
+      promises.push(this.#createBinLinks(node))
     }
 
     await promiseAllRejectLate(promises)
     process.emit('timeEnd', 'build:link')
   }
 
-  async [_createBinLinks] (node) {
+  async #createBinLinks (node) {
     if (this[_trashList].has(node.path)) {
       return
     }
@@ -420,11 +391,11 @@ module.exports = cls => class Builder extends cls {
       pkg: node.package,
       path: node.path,
       top: !!(node.isTop || node.globalTop),
-      force: this[_force],
+      force: this.options.force,
       global: !!node.globalTop,
     })
 
-    await (this[_doHandleOptionalFailure]
+    await (this.#doHandleOptionalFailure
       ? this[_handleOptionalFailure](node, p)
       : p)
 

package/lib/arborist/reify.js

@@ -24,13 +24,13 @@ const PackageJson = require('@npmcli/package-json')
 const packageContents = require('@npmcli/installed-package-contents')
 const runScript = require('@npmcli/run-script')
 const { checkEngine, checkPlatform } = require('npm-install-checks')
-const _force = Symbol.for('force')
 
 const treeCheck = require('../tree-check.js')
 const relpath = require('../relpath.js')
 const Diff = require('../diff.js')
 const retirePath = require('../retire-path.js')
 const promiseAllRejectLate = require('promise-all-reject-late')
+const { callLimit: promiseCallLimit } = require('promise-call-limit')
 const optionalSet = require('../optional-set.js')
 const calcDepFlags = require('../calc-dep-flags.js')
 const { saveTypeMap, hasSubKey } = require('../add-rm-pkg-deps.js')
@@ -47,8 +47,6 @@ const _retireShallowNodes = Symbol.for('retireShallowNodes')
 const _getBundlesByDepth = Symbol('getBundlesByDepth')
 const _registryResolved = Symbol('registryResolved')
 const _addNodeToTrashList = Symbol.for('addNodeToTrashList')
-const _workspaces = Symbol.for('workspaces')
-const _workspacesEnabled = Symbol.for('workspacesEnabled')
 
 // shared by rebuild mixin
 const _trashList = Symbol.for('trashList')
@@ -90,14 +88,11 @@ const _validateNodeModules = Symbol('validateNodeModules')
 const _nmValidated = Symbol('nmValidated')
 const _validatePath = Symbol('validatePath')
 const _reifyPackages = Symbol.for('reifyPackages')
-const _includeWorkspaceRoot = Symbol.for('includeWorkspaceRoot')
 
 const _omitDev = Symbol('omitDev')
 const _omitOptional = Symbol('omitOptional')
 const _omitPeer = Symbol('omitPeer')
 
-const _global = Symbol.for('global')
-
 const _pruneBundledMetadeps = Symbol('pruneBundledMetadeps')
 
 // defined by Ideal mixin
@@ -141,7 +136,7 @@ module.exports = cls => class Reifier extends cls {
   async reify (options = {}) {
     const linked = (options.installStrategy || this.options.installStrategy) === 'linked'
 
-    if (this[_packageLockOnly] && this[_global]) {
+    if (this[_packageLockOnly] && this.options.global) {
       const er = new Error('cannot generate lockfile for global packages')
       er.code = 'ESHRINKWRAPGLOBAL'
       throw er
@@ -286,7 +281,7 @@ module.exports = cls => class Reifier extends cls {
         .then(() => process.emit('timeEnd', 'reify:loadTrees'))
     }
 
-    const actualOpt = this[_global] ? {
+    const actualOpt = this.options.global ? {
       ignoreMissing: true,
       global: true,
       filter: (node, kid) => {
@@ -313,7 +308,7 @@ module.exports = cls => class Reifier extends cls {
       },
     } : { ignoreMissing: true }
 
-    if (!this[_global]) {
+    if (!this.options.global) {
       return Promise.all([
         this.loadActual(actualOpt),
         this.buildIdealTree(bitOpt),
@@ -340,12 +335,12 @@ module.exports = cls => class Reifier extends cls {
     // to just invalidate the parts that changed, but avoid walking the
     // whole tree again.
 
-    const includeWorkspaces = this[_workspacesEnabled]
-    const includeRootDeps = !this[_workspacesEnabled]
-      || this[_includeWorkspaceRoot] && this[_workspaces].length > 0
+    const includeWorkspaces = this.options.workspacesEnabled
+    const includeRootDeps = !includeWorkspaces
+      || this.options.includeWorkspaceRoot && this.options.workspaces.length > 0
 
     const filterNodes = []
-    if (this[_global] && this.explicitRequests.size) {
+    if (this.options.global && this.explicitRequests.size) {
       const idealTree = this.idealTree.target
       const actualTree = this.actualTree.target
       // we ONLY are allowed to make changes in the global top-level
@@ -363,7 +358,7 @@ module.exports = cls => class Reifier extends cls {
     } else {
       if (includeWorkspaces) {
         // add all ws nodes to filterNodes
-        for (const ws of this[_workspaces]) {
+        for (const ws of this.options.workspaces) {
           const ideal = this.idealTree.children.get(ws)
           if (ideal) {
             filterNodes.push(ideal)
@@ -655,7 +650,7 @@ module.exports = cls => class Reifier extends cls {
 
   // do not allow node_modules to be a symlink
   async [_validateNodeModules] (nm) {
-    if (this[_force] || this[_nmValidated].has(nm)) {
+    if (this.options.force || this[_nmValidated].has(nm)) {
       return
     }
     const st = await lstat(nm).catch(() => null)
@@ -817,10 +812,12 @@ module.exports = cls => class Reifier extends cls {
     }
 
     // extract all the nodes with bundles
-    return promiseAllRejectLate(set.map(node => {
-      this[_bundleUnpacked].add(node)
-      return this[_reifyNode](node)
-    }))
+    return promiseCallLimit(set.map(node => {
+      return () => {
+        this[_bundleUnpacked].add(node)
+        return this[_reifyNode](node)
+      }
+    }), { rejectLate: true })
     // then load their unpacked children and move into the ideal tree
       .then(nodes =>
         promiseAllRejectLate(nodes.map(async node => {
@@ -989,11 +986,11 @@ module.exports = cls => class Reifier extends cls {
     const tree = this.idealTree
 
     // if we're operating on a workspace, only audit the workspace deps
-    if (this[_workspaces] && this[_workspaces].length) {
+    if (this.options.workspaces.length) {
       options.filterSet = this.workspaceDependencySet(
         tree,
-        this[_workspaces],
-        this[_includeWorkspaceRoot]
+        this.options.workspaces,
+        this.options.includeWorkspaceRoot
       )
     }
 
@@ -1217,7 +1214,7 @@ module.exports = cls => class Reifier extends cls {
     // saveIdealTree to be able to write the lockfile by default.
     const saveIdealTree = !(
       (!save && !hasUpdates)
-      || this[_global]
+      || this.options.global
       || this[_dryRun]
     )
 
@@ -1563,7 +1560,7 @@ module.exports = cls => class Reifier extends cls {
     this.actualTree = this.idealTree
     this.idealTree = null
 
-    if (!this[_global]) {
+    if (!this.options.global) {
       await this.actualTree.meta.save()
       const ignoreScripts = !!this.options.ignoreScripts
       // if we aren't doing a dry run or ignoring scripts and we actually made changes to the dep
@@ -1590,4 +1587,22 @@ module.exports = cls => class Reifier extends cls {
       }
     }
   }
+
+  async dedupe (options = {}) {
+    // allow the user to set options on the ctor as well.
+    // XXX: deprecate separate method options objects.
+    options = { ...this.options, ...options }
+    const tree = await this.loadVirtual().catch(() => this.loadActual())
+    const names = []
+    for (const name of tree.inventory.query('name')) {
+      if (tree.inventory.query('name', name).size > 1) {
+        names.push(name)
+      }
+    }
+    return this.reify({
+      ...options,
+      preferDedupe: true,
+      update: { names },
+    })
+  }
 }

package/lib/case-insensitive-map.js

@@ -2,49 +2,49 @@
 // are case-insensitive and unicode-normalizing, so we need to treat
 // node.children.get('FOO') and node.children.get('foo') as the same thing.
 
-const _keys = Symbol('keys')
-const _normKey = Symbol('normKey')
-const normalize = s => s.normalize('NFKD').toLowerCase()
-const OGMap = Map
-module.exports = class Map extends OGMap {
+module.exports = class CIMap extends Map {
+  #keys = new Map()
+
   constructor (items = []) {
     super()
-    this[_keys] = new OGMap()
     for (const [key, val] of items) {
       this.set(key, val)
     }
   }
 
-  [_normKey] (key) {
-    return typeof key === 'string' ? normalize(key) : key
+  #normKey (key) {
+    if (typeof key !== 'string') {
+      return key
+    }
+    return key.normalize('NFKD').toLowerCase()
   }
 
   get (key) {
-    const normKey = this[_normKey](key)
-    return this[_keys].has(normKey) ? super.get(this[_keys].get(normKey))
+    const normKey = this.#normKey(key)
+    return this.#keys.has(normKey) ? super.get(this.#keys.get(normKey))
       : undefined
   }
 
   set (key, val) {
-    const normKey = this[_normKey](key)
-    if (this[_keys].has(normKey)) {
-      super.delete(this[_keys].get(normKey))
+    const normKey = this.#normKey(key)
+    if (this.#keys.has(normKey)) {
+      super.delete(this.#keys.get(normKey))
     }
-    this[_keys].set(normKey, key)
+    this.#keys.set(normKey, key)
     return super.set(key, val)
   }
 
   delete (key) {
-    const normKey = this[_normKey](key)
-    if (this[_keys].has(normKey)) {
-      const prevKey = this[_keys].get(normKey)
-      this[_keys].delete(normKey)
+    const normKey = this.#normKey(key)
+    if (this.#keys.has(normKey)) {
+      const prevKey = this.#keys.get(normKey)
+      this.#keys.delete(normKey)
       return super.delete(prevKey)
     }
   }
 
   has (key) {
-    const normKey = this[_normKey](key)
-    return this[_keys].has(normKey) && super.has(this[_keys].get(normKey))
+    const normKey = this.#normKey(key)
+    return this.#keys.has(normKey) && super.has(this.#keys.get(normKey))
   }
 }

package/lib/index.js

@@ -4,5 +4,3 @@ module.exports.Node = require('./node.js')
 module.exports.Link = require('./link.js')
 module.exports.Edge = require('./edge.js')
 module.exports.Shrinkwrap = require('./shrinkwrap.js')
-// XXX export the other classes, too.  shrinkwrap, diff, etc.
-// they're handy!