@npmcli/arborist 2.2.2 → 2.2.6

package/bin/lib/options.js

@@ -31,7 +31,9 @@ for (const arg of process.argv.slice(2)) {
   } else if (/^--omit=/.test(arg)) {
     options.omit = options.omit || []
     options.omit.push(arg.substr('--omit='.length))
-  } else if (/^--[^=]+=/.test(arg)) {
+  } else if (/^--before=/.test(arg))
+    options.before = new Date(arg.substr('--before='.length))
+  else if (/^--[^=]+=/.test(arg)) {
     const [key, ...v] = arg.replace(/^--/, '').split('=')
     const val = v.join('=')
     options[key] = val === 'false' ? false : val === 'true' ? true : val

package/lib/arborist/build-ideal-tree.js

@@ -398,6 +398,8 @@ module.exports = cls => class IdealTreeBuilder extends cls {
     if (this[_global] && (this[_updateAll] || this[_updateNames].length)) {
       const nm = resolve(this.path, 'node_modules')
       for (const name of await readdir(nm).catch(() => [])) {
+        if (this[_updateNames].includes(name))
+          this[_explicitRequests].add(name)
         tree.package.dependencies = tree.package.dependencies || {}
         if (this[_updateAll] || this[_updateNames].includes(name))
           tree.package.dependencies[name] = '*'
@@ -1121,8 +1123,11 @@ This is a one-time fix-up, please be patient...
       // we don't like it.  always fail strictly, always allow forcibly or
       // in non-strict mode if it's not our fault.  don't warn here, because
       // we are going to warn again when we place the deps, if we end up
-      // overriding for something else.
-      if (conflictOK)
+      // overriding for something else.  If the thing that has this dep
+      // isn't also required, then there's a good chance we won't need it,
+      // so allow it for now and let it conflict if it turns out to actually
+      // be necessary for the installation.
+      if (conflictOK || !required.has(edge.from))
         continue
 
       // ok, it's the root, or we're in unforced strict mode, so this is bad

package/lib/arborist/index.js

@@ -28,6 +28,7 @@
 
 const {resolve} = require('path')
 const {homedir} = require('os')
+const procLog = require('../proc-log.js')
 
 const mixins = [
   require('../tracker.js'),
@@ -54,6 +55,7 @@ class Arborist extends Base {
       path: options.path || '.',
       cache: options.cache || `${homedir()}/.npm/_cacache`,
       packumentCache: new Map(),
+      log: options.log || procLog,
     }
     this.cache = resolve(this.options.cache)
     this.path = resolve(this.options.path)

package/lib/arborist/reify.js

@@ -16,6 +16,7 @@ const mkdirp = require('mkdirp-infer-owner')
 const moveFile = require('@npmcli/move-file')
 const rimraf = promisify(require('rimraf'))
 const packageContents = require('@npmcli/installed-package-contents')
+const { checkEngine, checkPlatform } = require('npm-install-checks')
 
 const treeCheck = require('../tree-check.js')
 const relpath = require('../relpath.js')
@@ -43,6 +44,7 @@ const _loadTrees = Symbol.for('loadTrees')
 const _diffTrees = Symbol.for('diffTrees')
 const _createSparseTree = Symbol.for('createSparseTree')
 const _loadShrinkwrapsAndUpdateTrees = Symbol.for('loadShrinkwrapsAndUpdateTrees')
+const _shrinkwrapUnpacked = Symbol('shrinkwrapUnpacked')
 const _reifyNode = Symbol.for('reifyNode')
 const _extractOrLink = Symbol('extractOrLink')
 // defined by rebuild mixin
@@ -102,6 +104,7 @@ module.exports = cls => class Reifier extends cls {
 
     this.diff = null
     this[_retiredPaths] = {}
+    this[_shrinkwrapUnpacked] = new Set()
     this[_retiredUnchanged] = {}
     this[_sparseTreeDirs] = new Set()
     this[_sparseTreeRoots] = new Set()
@@ -233,9 +236,9 @@ module.exports = cls => class Reifier extends cls {
     const actualOpt = this[_global] ? {
       ignoreMissing: true,
       global: true,
-      filter: (node, kid) => this[_explicitRequests].size === 0 || !node.isProjectRoot
-        ? true
-        : (node.edgesOut.has(kid) || this[_explicitRequests].has(kid)),
+      filter: (node, kid) =>
+        this[_explicitRequests].size === 0 || !node.isProjectRoot ? true
+        : (this.idealTree.edgesOut.has(kid) || this[_explicitRequests].has(kid)),
     } : { ignoreMissing: true }
 
     if (!this[_global]) {
@@ -404,7 +407,8 @@ module.exports = cls => class Reifier extends cls {
   // shrinkwrap nodes define their dependency branches with a file, so
   // we need to unpack them, read that shrinkwrap file, and then update
   // the tree by calling loadVirtual with the node as the root.
-  [_loadShrinkwrapsAndUpdateTrees] (seen = new Set()) {
+  [_loadShrinkwrapsAndUpdateTrees] () {
+    const seen = this[_shrinkwrapUnpacked]
     const shrinkwraps = this.diff.leaves
       .filter(d => (d.action === 'CHANGE' || d.action === 'ADD') &&
         d.ideal.hasShrinkwrap && !seen.has(d.ideal) &&
@@ -428,6 +432,8 @@ module.exports = cls => class Reifier extends cls {
       // reload the diff and sparse tree because the ideal tree changed
       .then(() => this[_diffTrees]())
       .then(() => this[_createSparseTree]())
+      .then(() => this[_addOmitsToTrashList]())
+      .then(() => this[_loadShrinkwrapsAndUpdateTrees]())
       .then(() => process.emit('timeEnd', 'reify:loadShrinkwraps'))
   }
 
@@ -446,7 +452,19 @@ module.exports = cls => class Reifier extends cls {
     process.emit('time', timer)
     this.addTracker('reify', node.name, node.location)
 
+    const { npmVersion, nodeVersion } = this.options
     const p = Promise.resolve()
+      .then(() => {
+        // when we reify an optional node, check the engine and platform
+        // first. be sure to ignore the --force and --engine-strict flags,
+        // since we always want to skip any optional packages we can't install.
+        // these checks throwing will result in a rollback and removal
+        // of the mismatches
+        if (node.optional) {
+          checkEngine(node.package, npmVersion, nodeVersion, false)
+          checkPlatform(node.package, false)
+        }
+      })
       .then(() => this[_checkBins](node))
       .then(() => this[_extractOrLink](node))
       .then(() => this[_warnDeprecated](node))
@@ -718,7 +736,7 @@ module.exports = cls => class Reifier extends cls {
 
         const node = diff.ideal
         const bd = node.package.bundleDependencies
-        const sw = node.hasShrinkwrap
+        const sw = this[_shrinkwrapUnpacked].has(node)
 
         // check whether we still need to unpack this one.
         // test the inDepBundle last, since that's potentially a tree walk.

package/lib/audit-report.js

@@ -268,8 +268,8 @@ class AuditReport extends Map {
         id,
         url,
         title,
-        severity,
-        vulnerable_versions,
+        severity = 'high',
+        vulnerable_versions = '*',
         module_name: name,
       } = advisory
       bulk[name] = bulk[name] || []

package/lib/edge.js

@@ -87,16 +87,24 @@ class Edge {
 
   // return the edge data, and an explanation of how that edge came to be here
   [_explain] (seen) {
-    const { error, from } = this
+    const { error, from, bundled } = this
     return {
       type: this.type,
       name: this.name,
       spec: this.spec,
+      ...(bundled ? { bundled } : {}),
       ...(error ? { error } : {}),
       ...(from ? { from: from.explain(null, seen) } : {}),
     }
   }
 
+  get bundled () {
+    if (!this.from)
+      return false
+    const { package: { bundleDependencies = [] } } = this.from
+    return bundleDependencies.includes(this.name)
+  }
+
   get workspace () {
     return this[_type] === 'workspace'
   }

package/lib/node.js

@@ -731,7 +731,6 @@ class Node {
     // Note the subtle breaking change from v6: it is no longer possible
     // to have a different spec for a devDep than production dep.
     this[_loadDepType](this.package.optionalDependencies, 'optional')
-    this[_loadDepType](this.package.dependencies, 'prod')
 
     // Linked targets that are disconnected from the tree are tops,
     // but don't have a 'path' field, only a 'realpath', because we
@@ -755,6 +754,8 @@ class Node {
       this[_loadDepType](peerDependencies, 'peer')
       this[_loadDepType](peerOptional, 'peerOptional')
     }
+
+    this[_loadDepType](this.package.dependencies, 'prod')
   }
 
   [_loadDepType] (obj, type) {
@@ -763,8 +764,10 @@ class Node {
     for (const [name, spec] of Object.entries(obj || {})) {
       const accept = ad[name]
       // if it's already set, then we keep the existing edge
+      // Prod deps should not be marked as dev, however.
       // NB: the Edge ctor adds itself to from.edgesOut
-      if (!this.edgesOut.get(name))
+      const current = this.edgesOut.get(name)
+      if (!current || current.dev && type === 'prod')
         new Edge({ from, name, spec, accept, type })
     }
   }

package/lib/shrinkwrap.js

@@ -32,6 +32,7 @@ const mismatch = (a, b) => a && b && a !== b
 // After calling this.commit(), any nodes not present in the tree will have
 // been removed from the shrinkwrap data as well.
 
+const procLog = require('./proc-log.js')
 const YarnLock = require('./yarn-lock.js')
 const {promisify} = require('util')
 const rimraf = promisify(require('rimraf'))
@@ -39,7 +40,23 @@ const fs = require('fs')
 const readFile = promisify(fs.readFile)
 const writeFile = promisify(fs.writeFile)
 const stat = promisify(fs.stat)
-const readdir = promisify(fs.readdir)
+const readdir_ = promisify(fs.readdir)
+
+// XXX remove when drop support for node v10
+const lstat = promisify(fs.lstat)
+/* istanbul ignore next - version specific polyfill */
+const readdir = async (path, opt) => {
+  if (!opt || !opt.withFileTypes)
+    return readdir_(path, opt)
+  const ents = await readdir_(path, opt)
+  if (typeof ents[0] === 'string') {
+    return Promise.all(ents.map(async ent => {
+      return Object.assign(await lstat(path + '/' + ent), { name: ent })
+    }))
+  }
+  return ents
+}
+
 const { resolve, basename } = require('path')
 const specFromLock = require('./spec-from-lock.js')
 const versionFromTgz = require('./version-from-tgz.js')
@@ -59,6 +76,10 @@ const swKeyOrder = [
   'dependencies',
 ]
 
+// used to rewrite from yarn registry to npm registry
+const yarnRegRe = /^https?:\/\/registry.yarnpkg.com\//
+const npmRegRe = /^https?:\/\/registry.npmjs.org\//
+
 // sometimes resolved: is weird or broken, or something npa can't handle
 const specFromResolved = resolved => {
   try {
@@ -261,7 +282,10 @@ class Shrinkwrap {
       newline = '\n',
       shrinkwrapOnly = false,
       hiddenLockfile = false,
+      log = procLog,
     } = options
+
+    this.log = log
     this[_awaitingUpdate] = new Map()
     this.tree = null
     this.path = resolve(path || '.')
@@ -291,8 +315,6 @@ class Shrinkwrap {
     if (fromYarn && fromYarn.version) {
       // if it's the yarn or npm default registry, use the version as
       // our effective spec.  if it's any other kind of thing, use that.
-      const yarnRegRe = /^https?:\/\/registry.yarnpkg.com\//
-      const npmRegRe = /^https?:\/\/registry.npmjs.org\//
       const {resolved, version, integrity} = fromYarn
       const isYarnReg = spec.registry && yarnRegRe.test(resolved)
       const isnpmReg = spec.registry && !isYarnReg && npmRegRe.test(resolved)
@@ -396,6 +418,8 @@ class Shrinkwrap {
       // all good!  hidden lockfile is the newest thing in here.
       return data
     }).catch(er => {
+      const rel = relpath(this.path, this.filename)
+      this.log.verbose('shrinkwrap', `failed to load ${rel}`, er)
       this.loadingError = er
       this.loadedFromDisk = false
       this.ancientLockfile = false
@@ -733,6 +757,7 @@ class Shrinkwrap {
       : !/file:/.test(node.resolved) ? node.resolved
       : consistentResolve(node.resolved, node.path, this.path, true)
 
+    const spec = npa(`${node.name}@${edge.spec}`)
     const entry = this.yarnLock.entries.get(`${node.name}@${edge.spec}`)
 
     if (!entry ||
@@ -741,6 +766,9 @@ class Shrinkwrap {
         mismatch(pathFixed, entry.resolved))
       return
 
+    if (entry.resolved && yarnRegRe.test(entry.resolved) && spec.registry)
+      entry.resolved = entry.resolved.replace(yarnRegRe, 'https://registry.npmjs.org/')
+
     node.integrity = node.integrity || entry.integrity || null
     node.resolved = node.resolved ||
       consistentResolve(entry.resolved, this.path, node.path) || null

package/lib/tracker.js

@@ -1,13 +1,12 @@
-const procLog = require('./proc-log.js')
-
 const _progress = Symbol('_progress')
 const _onError = Symbol('_onError')
+const procLog = require('./proc-log.js')
 
 module.exports = cls => class Tracker extends cls {
   constructor (options = {}) {
     super(options)
-    this[_progress] = new Map()
     this.log = options.log || procLog
+    this[_progress] = new Map()
   }
 
   addTracker (section, subsection = null, key = null) {

package/lib/update-root-package-json.js

@@ -15,11 +15,18 @@ const depTypes = new Set([
   'peerDependencies',
 ])
 
+const parseJsonSafe = json => {
+  try {
+    return parseJSON(json)
+  } catch (er) {
+    return null
+  }
+}
+
 const updateRootPackageJson = async tree => {
   const filename = resolve(tree.path, 'package.json')
-  const originalContent = await readFile(filename, 'utf8')
-    .then(data => parseJSON(data))
-    .catch(() => null)
+  const originalJson = await readFile(filename, 'utf8').catch(() => null)
+  const originalContent = parseJsonSafe(originalJson)
 
   const depsData = orderDeps({
     ...tree.package,
@@ -36,12 +43,29 @@ const updateRootPackageJson = async tree => {
   }
 
   // if there's no package.json, just use internal pkg info as source of truth
-  const packageJsonContent = originalContent || depsData
+  // clone the object though, so we can still refer to what it originally was
+  const packageJsonContent = !originalContent ? depsData
+    : Object.assign({}, originalContent)
 
   // loop through all types of dependencies and update package json content
   for (const type of depTypes)
     packageJsonContent[type] = depsData[type]
 
+  // if original package.json had dep in peerDeps AND deps, preserve that.
+  const { dependencies: origProd, peerDependencies: origPeer } =
+    originalContent || {}
+  const { peerDependencies: newPeer } = packageJsonContent
+  if (origProd && origPeer && newPeer) {
+    // we have original prod/peer deps, and new peer deps
+    // copy over any that were in both in the original
+    for (const name of Object.keys(origPeer)) {
+      if (origProd[name] !== undefined && newPeer[name] !== undefined) {
+        packageJsonContent.dependencies = packageJsonContent.dependencies || {}
+        packageJsonContent.dependencies[name] = newPeer[name]
+      }
+    }
+  }
+
   // format content
   const {
     [Symbol.for('indent')]: indent,
@@ -52,7 +76,8 @@ const updateRootPackageJson = async tree => {
   const content = (JSON.stringify(packageJsonContent, null, format) + '\n')
     .replace(/\n/g, eol)
 
-  return writeFile(filename, content)
+  if (content !== originalJson)
+    return writeFile(filename, content)
 }
 
 module.exports = updateRootPackageJson

package/package.json

@@ -1,11 +1,11 @@
 {
   "name": "@npmcli/arborist",
-  "version": "2.2.2",
+  "version": "2.2.6",
   "description": "Manage node_modules trees",
   "dependencies": {
-    "@npmcli/installed-package-contents": "^1.0.6",
+    "@npmcli/installed-package-contents": "^1.0.7",
     "@npmcli/map-workspaces": "^1.0.2",
-    "@npmcli/metavuln-calculator": "^1.0.1",
+    "@npmcli/metavuln-calculator": "^1.1.0",
     "@npmcli/move-file": "^1.1.0",
     "@npmcli/name-from-folder": "^1.0.1",
     "@npmcli/node-gyp": "^1.0.1",
@@ -24,7 +24,7 @@
     "parse-conflict-json": "^1.1.1",
     "promise-all-reject-late": "^1.0.0",
     "promise-call-limit": "^1.0.1",
-    "read-package-json-fast": "^2.0.1",
+    "read-package-json-fast": "^2.0.2",
     "readdir-scoped-modules": "^1.1.0",
     "semver": "^7.3.4",
     "tar": "^6.1.0",
@@ -46,7 +46,8 @@
     "tcompare": "^3.0.4"
   },
   "scripts": {
-    "test": "tap",
+    "test": "npm run test-only --",
+    "test-only": "tap",
     "posttest": "npm run lint",
     "snap": "tap",
     "postsnap": "npm run lint",
@@ -76,12 +77,16 @@
   },
   "tap": {
     "100": true,
-    "node-arg": [
-      "--unhandled-rejections=strict"
-    ],
     "after": "test/fixtures/cleanup.js",
     "coverage-map": "map.js",
     "esm": false,
+    "test-env": [
+      "NODE_OPTIONS=--no-warnings"
+    ],
+    "node-arg": [
+      "--no-warnings",
+      "--no-deprecation"
+    ],
     "timeout": "120"
   }
 }