@npmcli/arborist 7.3.0 → 7.4.0

package/lib/query-selector-all.js

@@ -8,6 +8,7 @@ const { minimatch } = require('minimatch')
 const npa = require('npm-package-arg')
 const pacote = require('pacote')
 const semver = require('semver')
+const fetch = require('npm-registry-fetch')
 
 // handle results for parsed query asts, results are stored in a map that has a
 // key that points to each ast selector node and stores the resulting array of
@@ -18,6 +19,7 @@ class Results {
   #initialItems
   #inventory
   #outdatedCache = new Map()
+  #vulnCache
   #pendingCombinator
   #results = new Map()
   #targetNode
@@ -26,6 +28,7 @@ class Results {
     this.#currentAstSelector = opts.rootAstNode.nodes[0]
     this.#inventory = opts.inventory
     this.#initialItems = opts.initialItems
+    this.#vulnCache = opts.vulnCache
     this.#targetNode = opts.targetNode
 
     this.currentResults = this.#initialItems
@@ -211,6 +214,7 @@ class Results {
         inventory: this.#inventory,
         rootAstNode: this.currentAstNode.nestedNode,
         targetNode: item,
+        vulnCache: this.#vulnCache,
       })
       if (res.size > 0) {
         found.push(item)
@@ -239,6 +243,7 @@ class Results {
       inventory: this.#inventory,
       rootAstNode: this.currentAstNode.nestedNode,
       targetNode: this.currentAstNode,
+      vulnCache: this.#vulnCache,
     })
     return [...res]
   }
@@ -266,6 +271,7 @@ class Results {
       inventory: this.#inventory,
       rootAstNode: this.currentAstNode.nestedNode,
       targetNode: this.currentAstNode,
+      vulnCache: this.#vulnCache,
     })
     const internalSelector = new Set(res)
     return this.initialItems.filter(node =>
@@ -432,6 +438,75 @@ class Results {
     return this.initialItems.filter(node => node.target.edgesIn.size > 1)
   }
 
+  async vulnPseudo () {
+    if (!this.initialItems.length) {
+      return this.initialItems
+    }
+    if (!this.#vulnCache) {
+      const packages = {}
+      // We have to map the items twice, once to get the request, and a second time to filter out the results of that request
+      this.initialItems.map((node) => {
+        if (node.isProjectRoot || node.package.private) {
+          return
+        }
+        if (!packages[node.name]) {
+          packages[node.name] = []
+        }
+        if (!packages[node.name].includes(node.version)) {
+          packages[node.name].push(node.version)
+        }
+      })
+      const res = await fetch('/-/npm/v1/security/advisories/bulk', {
+        ...this.flatOptions,
+        registry: this.flatOptions.auditRegistry || this.flatOptions.registry,
+        method: 'POST',
+        gzip: true,
+        body: packages,
+      })
+      this.#vulnCache = await res.json()
+    }
+    const advisories = this.#vulnCache
+    const { vulns } = this.currentAstNode
+    return this.initialItems.filter(item => {
+      const vulnerable = advisories[item.name]?.filter(advisory => {
+        // This could be for another version of this package elsewhere in the tree
+        if (!semver.intersects(advisory.vulnerable_versions, item.version)) {
+          return false
+        }
+        if (!vulns) {
+          return true
+        }
+        // vulns are OR with each other, if any one matches we're done
+        for (const vuln of vulns) {
+          if (vuln.severity && !vuln.severity.includes('*')) {
+            if (!vuln.severity.includes(advisory.severity)) {
+              continue
+            }
+          }
+
+          if (vuln?.cwe) {
+            // * is special, it means "has a cwe"
+            if (vuln.cwe.includes('*')) {
+              if (!advisory.cwe.length) {
+                continue
+              }
+            } else if (!vuln.cwe.every(cwe => advisory.cwe.includes(`CWE-${cwe}`))) {
+              continue
+            }
+          }
+          return true
+        }
+      })
+      if (vulnerable?.length) {
+        item.queryContext = {
+          advisories: vulnerable,
+        }
+        return true
+      }
+      return false
+    })
+  }
+
   async outdatedPseudo () {
     const { outdatedKind = 'any' } = this.currentAstNode
 
@@ -445,6 +520,11 @@ class Results {
         return false
       }
 
+      // private packages can't be published, skip them
+      if (node.package.private) {
+        return false
+      }
+
       // we cache the promise representing the full versions list, this helps reduce the
       // number of requests we send by keeping population of the cache in a single tick
       // making it less likely that multiple requests for the same package will be inflight
@@ -839,8 +919,6 @@ const retrieveNodesFromParsedAst = async (opts) => {
   return results.collect(rootAstNode)
 }
 
-// We are keeping this async in the event that we do add async operators, we
-// won't have to have a breaking change on this function signature.
 const querySelectorAll = async (targetNode, query, flatOptions) => {
   // This never changes ever we just pass it around. But we can't scope it to
   // this whole file if we ever want to support concurrent calls to this

package/lib/tracker.js

@@ -1,47 +1,46 @@
-const _progress = Symbol('_progress')
-const _onError = Symbol('_onError')
-const _setProgress = Symbol('_setProgess')
 const npmlog = require('npmlog')
 
 module.exports = cls => class Tracker extends cls {
+  #progress = new Map()
+  #setProgress
+
   constructor (options = {}) {
     super(options)
-    this[_setProgress] = !!options.progress
-    this[_progress] = new Map()
+    this.#setProgress = !!options.progress
   }
 
   addTracker (section, subsection = null, key = null) {
     if (section === null || section === undefined) {
-      this[_onError](`Tracker can't be null or undefined`)
+      this.#onError(`Tracker can't be null or undefined`)
     }
 
     if (key === null) {
       key = subsection
     }
 
-    const hasTracker = this[_progress].has(section)
-    const hasSubtracker = this[_progress].has(`${section}:${key}`)
+    const hasTracker = this.#progress.has(section)
+    const hasSubtracker = this.#progress.has(`${section}:${key}`)
 
     if (hasTracker && subsection === null) {
       // 0. existing tracker, no subsection
-      this[_onError](`Tracker "${section}" already exists`)
+      this.#onError(`Tracker "${section}" already exists`)
     } else if (!hasTracker && subsection === null) {
       // 1. no existing tracker, no subsection
       // Create a new tracker from npmlog
       // starts progress bar
-      if (this[_setProgress] && this[_progress].size === 0) {
+      if (this.#setProgress && this.#progress.size === 0) {
         npmlog.enableProgress()
       }
 
-      this[_progress].set(section, npmlog.newGroup(section))
+      this.#progress.set(section, npmlog.newGroup(section))
     } else if (!hasTracker && subsection !== null) {
       // 2. no parent tracker and subsection
-      this[_onError](`Parent tracker "${section}" does not exist`)
+      this.#onError(`Parent tracker "${section}" does not exist`)
     } else if (!hasTracker || !hasSubtracker) {
       // 3. existing parent tracker, no subsection tracker
-      // Create a new subtracker in this[_progress] from parent tracker
-      this[_progress].set(`${section}:${key}`,
-        this[_progress].get(section).newGroup(`${section}:${subsection}`)
+      // Create a new subtracker in this.#progress from parent tracker
+      this.#progress.set(`${section}:${key}`,
+        this.#progress.get(section).newGroup(`${section}:${subsection}`)
       )
     }
     // 4. existing parent tracker, existing subsection tracker
@@ -50,22 +49,22 @@ module.exports = cls => class Tracker extends cls {
 
   finishTracker (section, subsection = null, key = null) {
     if (section === null || section === undefined) {
-      this[_onError](`Tracker can't be null or undefined`)
+      this.#onError(`Tracker can't be null or undefined`)
     }
 
     if (key === null) {
       key = subsection
     }
 
-    const hasTracker = this[_progress].has(section)
-    const hasSubtracker = this[_progress].has(`${section}:${key}`)
+    const hasTracker = this.#progress.has(section)
+    const hasSubtracker = this.#progress.has(`${section}:${key}`)
 
     // 0. parent tracker exists, no subsection
-    // Finish parent tracker and remove from this[_progress]
+    // Finish parent tracker and remove from this.#progress
     if (hasTracker && subsection === null) {
       // check if parent tracker does
       // not have any remaining children
-      const keys = this[_progress].keys()
+      const keys = this.#progress.keys()
       for (const key of keys) {
         if (key.match(new RegExp(section + ':'))) {
           this.finishTracker(section, key)
@@ -73,28 +72,28 @@ module.exports = cls => class Tracker extends cls {
       }
 
       // remove parent tracker
-      this[_progress].get(section).finish()
-      this[_progress].delete(section)
+      this.#progress.get(section).finish()
+      this.#progress.delete(section)
 
       // remove progress bar if all
       // trackers are finished
-      if (this[_setProgress] && this[_progress].size === 0) {
+      if (this.#setProgress && this.#progress.size === 0) {
         npmlog.disableProgress()
       }
     } else if (!hasTracker && subsection === null) {
       // 1. no existing parent tracker, no subsection
-      this[_onError](`Tracker "${section}" does not exist`)
+      this.#onError(`Tracker "${section}" does not exist`)
     } else if (!hasTracker || hasSubtracker) {
       // 2. subtracker exists
-      // Finish subtracker and remove from this[_progress]
-      this[_progress].get(`${section}:${key}`).finish()
-      this[_progress].delete(`${section}:${key}`)
+      // Finish subtracker and remove from this.#progress
+      this.#progress.get(`${section}:${key}`).finish()
+      this.#progress.delete(`${section}:${key}`)
     }
     // 3. existing parent tracker, no subsection
   }
 
-  [_onError] (msg) {
-    if (this[_setProgress]) {
+  #onError (msg) {
+    if (this.#setProgress) {
       npmlog.disableProgress()
     }
     throw new Error(msg)

package/lib/version-from-tgz.js

@@ -1,22 +1,21 @@
-/* eslint node/no-deprecated-api: "off" */
 const semver = require('semver')
 const { basename } = require('path')
-const { parse } = require('url')
+const { URL } = require('url')
 module.exports = (name, tgz) => {
   const base = basename(tgz)
   if (!base.endsWith('.tgz')) {
     return null
   }
 
-  const u = parse(tgz)
-  if (/^https?:/.test(u.protocol)) {
+  if (tgz.startsWith('http:/') || tgz.startsWith('https:/')) {
+    const u = new URL(tgz)
     // registry url?  check for most likely pattern.
     // either /@foo/bar/-/bar-1.2.3.tgz or
     // /foo/-/foo-1.2.3.tgz, and fall through to
     // basename checking.  Note that registries can
     // be mounted below the root url, so /a/b/-/x/y/foo/-/foo-1.2.3.tgz
     // is a potential option.
-    const tfsplit = u.path.slice(1).split('/-/')
+    const tfsplit = u.pathname.slice(1).split('/-/')
     if (tfsplit.length > 1) {
       const afterTF = tfsplit.pop()
       if (afterTF === base) {

package/lib/vuln.js

@@ -16,24 +16,23 @@ const semverOpt = { loose: true, includePrerelease: true }
 
 const localeCompare = require('@isaacs/string-locale-compare')('en')
 const npa = require('npm-package-arg')
-const _range = Symbol('_range')
-const _simpleRange = Symbol('_simpleRange')
-const _fixAvailable = Symbol('_fixAvailable')
 
 const severities = new Map([
-  ['info', 0],
-  ['low', 1],
-  ['moderate', 2],
-  ['high', 3],
-  ['critical', 4],
-  [null, -1],
+  ['info', 0], [0, 'info'],
+  ['low', 1], [1, 'low'],
+  ['moderate', 2], [2, 'moderate'],
+  ['high', 3], [3, 'high'],
+  ['critical', 4], [4, 'critical'],
+  [null, -1], [-1, null],
 ])
 
-for (const [name, val] of severities.entries()) {
-  severities.set(val, name)
-}
-
 class Vuln {
+  #range = null
+  #simpleRange = null
+  // assume a fix is available unless it hits a top node
+  // that locks it in place, setting this false or {isSemVerMajor, version}.
+  #fixAvailable = true
+
   constructor ({ name, advisory }) {
     this.name = name
     this.via = new Set()
@@ -41,23 +40,18 @@ class Vuln {
     this.severity = null
     this.effects = new Set()
     this.topNodes = new Set()
-    this[_range] = null
-    this[_simpleRange] = null
     this.nodes = new Set()
-    // assume a fix is available unless it hits a top node
-    // that locks it in place, setting this false or {isSemVerMajor, version}.
-    this[_fixAvailable] = true
     this.addAdvisory(advisory)
     this.packument = advisory.packument
     this.versions = advisory.versions
   }
 
   get fixAvailable () {
-    return this[_fixAvailable]
+    return this.#fixAvailable
   }
 
   set fixAvailable (f) {
-    this[_fixAvailable] = f
+    this.#fixAvailable = f
     // if there's a fix available for this at the top level, it means that
     // it will also fix the vulns that led to it being there.  to get there,
     // we set the vias to the most "strict" of fix availables.
@@ -131,7 +125,7 @@ class Vuln {
       effects: [...this.effects].map(v => v.name).sort(localeCompare),
       range: this.simpleRange,
       nodes: [...this.nodes].map(n => n.location).sort(localeCompare),
-      fixAvailable: this[_fixAvailable],
+      fixAvailable: this.#fixAvailable,
     }
   }
 
@@ -151,8 +145,8 @@ class Vuln {
     this.advisories.delete(advisory)
     // make sure we have the max severity of all the vulns causing this one
     this.severity = null
-    this[_range] = null
-    this[_simpleRange] = null
+    this.#range = null
+    this.#simpleRange = null
     // refresh severity
     for (const advisory of this.advisories) {
       this.addAdvisory(advisory)
@@ -170,27 +164,30 @@ class Vuln {
   addAdvisory (advisory) {
     this.advisories.add(advisory)
     const sev = severities.get(advisory.severity)
-    this[_range] = null
-    this[_simpleRange] = null
+    this.#range = null
+    this.#simpleRange = null
     if (sev > severities.get(this.severity)) {
       this.severity = advisory.severity
     }
   }
 
   get range () {
-    return this[_range] ||
-      (this[_range] = [...this.advisories].map(v => v.range).join(' || '))
+    if (!this.#range) {
+      this.#range = [...this.advisories].map(v => v.range).join(' || ')
+    }
+    return this.#range
   }
 
   get simpleRange () {
-    if (this[_simpleRange] && this[_simpleRange] === this[_range]) {
-      return this[_simpleRange]
+    if (this.#simpleRange && this.#simpleRange === this.#range) {
+      return this.#simpleRange
     }
 
     const versions = [...this.advisories][0].versions
     const range = this.range
-    const simple = simplifyRange(versions, range, semverOpt)
-    return this[_simpleRange] = this[_range] = simple
+    this.#simpleRange = simplifyRange(versions, range, semverOpt)
+    this.#range = this.#simpleRange
+    return this.#simpleRange
   }
 
   isVulnerable (node) {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@npmcli/arborist",
-  "version": "7.3.0",
+  "version": "7.4.0",
   "description": "Manage node_modules trees",
   "dependencies": {
     "@isaacs/string-locale-compare": "^1.1.0",
@@ -11,7 +11,7 @@
     "@npmcli/name-from-folder": "^2.0.0",
     "@npmcli/node-gyp": "^3.0.0",
     "@npmcli/package-json": "^5.0.0",
-    "@npmcli/query": "^3.0.1",
+    "@npmcli/query": "^3.1.0",
     "@npmcli/run-script": "^7.0.2",
     "bin-links": "^4.0.1",
     "cacache": "^18.0.0",
@@ -30,7 +30,7 @@
     "parse-conflict-json": "^3.0.0",
     "proc-log": "^3.0.0",
     "promise-all-reject-late": "^1.0.0",
-    "promise-call-limit": "^1.0.2",
+    "promise-call-limit": "^3.0.1",
     "read-package-json-fast": "^3.0.2",
     "semver": "^7.3.7",
     "ssri": "^10.0.5",

package/lib/arborist/audit.js

@@ -1,51 +0,0 @@
-// mixin implementing the audit method
-
-const AuditReport = require('../audit-report.js')
-
-// shared with reify
-const _global = Symbol.for('global')
-const _workspaces = Symbol.for('workspaces')
-const _includeWorkspaceRoot = Symbol.for('includeWorkspaceRoot')
-
-module.exports = cls => class Auditor extends cls {
-  async audit (options = {}) {
-    this.addTracker('audit')
-    if (this[_global]) {
-      throw Object.assign(
-        new Error('`npm audit` does not support testing globals'),
-        { code: 'EAUDITGLOBAL' }
-      )
-    }
-
-    // allow the user to set options on the ctor as well.
-    // XXX: deprecate separate method options objects.
-    options = { ...this.options, ...options }
-
-    process.emit('time', 'audit')
-    let tree
-    if (options.packageLock === false) {
-      // build ideal tree
-      await this.loadActual(options)
-      await this.buildIdealTree()
-      tree = this.idealTree
-    } else {
-      tree = await this.loadVirtual()
-    }
-    if (this[_workspaces] && this[_workspaces].length) {
-      options.filterSet = this.workspaceDependencySet(
-        tree,
-        this[_workspaces],
-        this[_includeWorkspaceRoot]
-      )
-    }
-    if (!options.workspacesEnabled) {
-      options.filterSet =
-        this.excludeWorkspacesDependencySet(tree)
-    }
-    this.auditReport = await AuditReport.load(tree, options)
-    const ret = options.fix ? this.reify(options) : this.auditReport
-    process.emit('timeEnd', 'audit')
-    this.finishTracker('audit')
-    return ret
-  }
-}

package/lib/arborist/deduper.js

@@ -1,19 +0,0 @@
-module.exports = cls => class Deduper extends cls {
-  async dedupe (options = {}) {
-    // allow the user to set options on the ctor as well.
-    // XXX: deprecate separate method options objects.
-    options = { ...this.options, ...options }
-    const tree = await this.loadVirtual().catch(() => this.loadActual())
-    const names = []
-    for (const name of tree.inventory.query('name')) {
-      if (tree.inventory.query('name', name).size > 1) {
-        names.push(name)
-      }
-    }
-    return this.reify({
-      ...options,
-      preferDedupe: true,
-      update: { names },
-    })
-  }
-}

package/lib/arborist/pruner.js

@@ -1,30 +0,0 @@
-const _idealTreePrune = Symbol.for('idealTreePrune')
-const _workspacesEnabled = Symbol.for('workspacesEnabled')
-const _addNodeToTrashList = Symbol.for('addNodeToTrashList')
-
-module.exports = cls => class Pruner extends cls {
-  async prune (options = {}) {
-    // allow the user to set options on the ctor as well.
-    // XXX: deprecate separate method options objects.
-    options = { ...this.options, ...options }
-
-    await this.buildIdealTree(options)
-
-    this[_idealTreePrune]()
-
-    if (!this[_workspacesEnabled]) {
-      const excludeNodes = this.excludeWorkspacesDependencySet(this.idealTree)
-      for (const node of this.idealTree.inventory.values()) {
-        if (
-          node.parent !== null
-          && !node.isProjectRoot
-          && !excludeNodes.has(node)
-        ) {
-          this[_addNodeToTrashList](node)
-        }
-      }
-    }
-
-    return this.reify(options)
-  }
-}

package/lib/arborist/set-workspaces.js

@@ -1,19 +0,0 @@
-const mapWorkspaces = require('@npmcli/map-workspaces')
-
-// shared ref used by other mixins/Arborist
-const _setWorkspaces = Symbol.for('setWorkspaces')
-
-module.exports = cls => class MapWorkspaces extends cls {
-  async [_setWorkspaces] (node) {
-    const workspaces = await mapWorkspaces({
-      cwd: node.path,
-      pkg: node.package,
-    })
-
-    if (node && workspaces.size) {
-      node.workspaces = workspaces
-    }
-
-    return node
-  }
-}

package/lib/get-workspace-nodes.js

@@ -1,36 +0,0 @@
-// Get the actual nodes corresponding to a root node's child workspaces,
-// given a list of workspace names.
-
-const log = require('proc-log')
-const relpath = require('./relpath.js')
-
-const getWorkspaceNodes = (tree, workspaces) => {
-  const wsMap = tree.workspaces
-  if (!wsMap) {
-    log.warn('workspaces', 'filter set, but no workspaces present')
-    return []
-  }
-
-  const nodes = []
-  for (const name of workspaces) {
-    const path = wsMap.get(name)
-    if (!path) {
-      log.warn('workspaces', `${name} in filter set, but not in workspaces`)
-      continue
-    }
-
-    const loc = relpath(tree.realpath, path)
-    const node = tree.inventory.get(loc)
-
-    if (!node) {
-      log.warn('workspaces', `${name} in filter set, but no workspace folder present`)
-      continue
-    }
-
-    nodes.push(node)
-  }
-
-  return nodes
-}
-
-module.exports = getWorkspaceNodes