@npmcli/arborist 7.5.1 → 7.5.3

package/bin/index.js

@@ -1,7 +1,7 @@
 #!/usr/bin/env node
 
-const fs = require('fs')
-const path = require('path')
+const fs = require('node:fs')
+const path = require('node:path')
 const { time } = require('proc-log')
 
 const { bin, arb: options } = require('./lib/options')

package/bin/lib/logging.js

@@ -1,8 +1,8 @@
 const { log } = require('proc-log')
-const fs = require('fs')
-const { dirname } = require('path')
-const os = require('os')
-const { inspect, format } = require('util')
+const fs = require('node:fs')
+const { dirname } = require('node:path')
+const os = require('node:os')
+const { inspect, format } = require('node:util')
 
 const { bin: options } = require('./options.js')
 

package/bin/lib/options.js

@@ -1,5 +1,5 @@
 const nopt = require('nopt')
-const path = require('path')
+const path = require('node:path')
 
 const has = (o, k) => Object.prototype.hasOwnProperty.call(o, k)
 

package/bin/lib/print-tree.js

@@ -1,4 +1,4 @@
-const { inspect } = require('util')
+const { inspect } = require('node:util')
 const log = require('./logging.js')
 
 module.exports = tree => log.info(inspect(tree.toJSON(), { depth: Infinity }))

package/lib/arborist/build-ideal-tree.js

@@ -6,10 +6,10 @@ const pacote = require('pacote')
 const cacache = require('cacache')
 const { callLimit: promiseCallLimit } = require('promise-call-limit')
 const realpath = require('../../lib/realpath.js')
-const { resolve, dirname } = require('path')
+const { resolve, dirname } = require('node:path')
 const treeCheck = require('../tree-check.js')
 const { readdirScoped } = require('@npmcli/fs')
-const { lstat, readlink } = require('fs/promises')
+const { lstat, readlink } = require('node:fs/promises')
 const { depth } = require('treeverse')
 const { log, time } = require('proc-log')
 const { redact } = require('@npmcli/redact')
@@ -53,48 +53,26 @@ const _addNodeToTrashList = Symbol.for('addNodeToTrashList')
 // they'll affect things deeper in, then alphabetical for consistency between
 // installs
 class DepsQueue {
-  // [{ sorted, items }] indexed by depth
   #deps = []
   #sorted = true
-  #minDepth = 0
-  #length = 0
 
   get length () {
-    return this.#length
+    return this.#deps.length
   }
 
   push (item) {
-    if (!this.#deps[item.depth]) {
-      this.#length++
-      this.#deps[item.depth] = { sorted: true, items: [item] }
-      // no minDepth check needed, this branch is only reached when we are in
-      // the middle of a shallower depth and creating a new one
-      return
-    }
-    if (!this.#deps[item.depth].items.includes(item)) {
-      this.#length++
-      this.#deps[item.depth].sorted = false
-      this.#deps[item.depth].items.push(item)
-      if (item.depth < this.#minDepth) {
-        this.#minDepth = item.depth
-      }
+    if (!this.#deps.includes(item)) {
+      this.#sorted = false
+      this.#deps.push(item)
     }
   }
 
   pop () {
-    let depth
-    while (!depth?.items.length) {
-      depth = this.#deps[this.#minDepth]
-      if (!depth?.items.length) {
-        this.#minDepth++
-      }
+    if (!this.#sorted) {
+      this.#deps.sort((a, b) => (a.depth - b.depth) || localeCompare(a.path, b.path))
+      this.#sorted = true
     }
-    if (!depth.sorted) {
-      depth.items.sort((a, b) => localeCompare(a.path, b.path))
-      depth.sorted = true
-    }
-    this.#length--
-    return depth.items.shift()
+    return this.#deps.shift()
   }
 }
 
@@ -1022,6 +1000,10 @@ This is a one-time fix-up, please be patient...
           // may well be an optional dep that has gone missing.  it'll
           // fail later anyway.
           for (const e of this.#problemEdges(placed)) {
+            // XXX This is somehow load bearing.  This makes tests that print
+            // the ideal tree of a tree with tarball dependencies fail. This
+            // can't be changed or removed till we figure out why
+            // The test is named "tarball deps with transitive tarball deps"
             promises.push(() =>
               this.#fetchManifest(npa.resolve(e.name, e.spec, fromPath(placed, e)))
                 .catch(() => null)
@@ -1204,6 +1186,7 @@ This is a one-time fix-up, please be patient...
     const options = {
       ...this.options,
       avoid: this.#avoidRange(spec.name),
+      fullMetadata: true,
     }
     // get the intended spec and stored metadata from yarn.lock file,
     // if available and valid.
@@ -1212,19 +1195,10 @@ This is a one-time fix-up, please be patient...
     if (this.#manifests.has(spec.raw)) {
       return this.#manifests.get(spec.raw)
     } else {
-      const cleanRawSpec = redact(spec.rawSpec)
-      log.silly('fetch manifest', spec.raw.replace(spec.rawSpec, cleanRawSpec))
-      const o = {
-        ...options,
-        fullMetadata: true,
-      }
-      const p = pacote.manifest(spec, o)
-        .then(({ license, ...mani }) => {
-          this.#manifests.set(spec.raw, mani)
-          return mani
-        })
-      this.#manifests.set(spec.raw, p)
-      return p
+      log.silly('fetch manifest', spec.raw.replace(spec.rawSpec, redact(spec.rawSpec)))
+      const mani = await pacote.manifest(spec, options)
+      this.#manifests.set(spec.raw, mani)
+      return mani
     }
   }
 

package/lib/arborist/index.js

@@ -26,15 +26,15 @@
 // the base class, so that the overall voltron class is easier to test and
 // cover, and separation of concerns can be maintained.
 
-const { resolve } = require('path')
-const { homedir } = require('os')
+const { resolve } = require('node:path')
+const { homedir } = require('node:os')
 const { depth } = require('treeverse')
 const mapWorkspaces = require('@npmcli/map-workspaces')
 const { log, time } = require('proc-log')
-
 const { saveTypeMap } = require('../add-rm-pkg-deps.js')
 const AuditReport = require('../audit-report.js')
 const relpath = require('../relpath.js')
+const PackumentCache = require('../packument-cache.js')
 
 const mixins = [
   require('../tracker.js'),
@@ -47,7 +47,7 @@ const mixins = [
 ]
 
 const _setWorkspaces = Symbol.for('setWorkspaces')
-const Base = mixins.reduce((a, b) => b(a), require('events'))
+const Base = mixins.reduce((a, b) => b(a), require('node:events'))
 
 // if it's 1, 2, or 3, set it explicitly that.
 // if undefined or null, set it null
@@ -82,7 +82,7 @@ class Arborist extends Base {
       installStrategy: options.global ? 'shallow' : (options.installStrategy ? options.installStrategy : 'hoisted'),
       lockfileVersion: lockfileVersion(options.lockfileVersion),
       packageLockOnly: !!options.packageLockOnly,
-      packumentCache: options.packumentCache || new Map(),
+      packumentCache: options.packumentCache || new PackumentCache(),
       path: options.path || '.',
       rebuildBundle: 'rebuildBundle' in options ? !!options.rebuildBundle : true,
       replaceRegistryHost: options.replaceRegistryHost,

package/lib/arborist/isolated-reifier.js

@@ -1,11 +1,11 @@
 const _makeIdealGraph = Symbol('makeIdealGraph')
 const _createIsolatedTree = Symbol.for('createIsolatedTree')
 const _createBundledTree = Symbol('createBundledTree')
-const { mkdirSync } = require('fs')
+const { mkdirSync } = require('node:fs')
 const pacote = require('pacote')
-const { join } = require('path')
+const { join } = require('node:path')
 const { depth } = require('treeverse')
-const crypto = require('crypto')
+const crypto = require('node:crypto')
 
 // cache complicated function results
 const memoize = (fn) => {

package/lib/arborist/load-actual.js

@@ -1,6 +1,6 @@
 // mix-in implementing the loadActual method
 
-const { relative, dirname, resolve, join, normalize } = require('path')
+const { relative, dirname, resolve, join, normalize } = require('node:path')
 
 const rpj = require('read-package-json-fast')
 const { readdirScoped } = require('@npmcli/fs')

package/lib/arborist/load-virtual.js

@@ -1,7 +1,7 @@
 // mixin providing the loadVirtual method
 const mapWorkspaces = require('@npmcli/map-workspaces')
 
-const { resolve } = require('path')
+const { resolve } = require('node:path')
 
 const nameFromFolder = require('@npmcli/name-from-folder')
 const consistentResolve = require('../consistent-resolve.js')

package/lib/arborist/rebuild.js

@@ -8,7 +8,7 @@ const rpj = require('read-package-json-fast')
 const binLinks = require('bin-links')
 const runScript = require('@npmcli/run-script')
 const { callLimit: promiseCallLimit } = require('promise-call-limit')
-const { resolve } = require('path')
+const { resolve } = require('node:path')
 const { isNodeGypPackage, defaultGypInstallScript } = require('@npmcli/node-gyp')
 const { log, time } = require('proc-log')
 

package/lib/arborist/reify.js

@@ -11,14 +11,14 @@ const { log, time } = require('proc-log')
 const hgi = require('hosted-git-info')
 const rpj = require('read-package-json-fast')
 
-const { dirname, resolve, relative, join } = require('path')
+const { dirname, resolve, relative, join } = require('node:path')
 const { depth: dfwalk } = require('treeverse')
 const {
   lstat,
   mkdir,
   rm,
   symlink,
-} = require('fs/promises')
+} = require('node:fs/promises')
 const { moveFile } = require('@npmcli/fs')
 const PackageJson = require('@npmcli/package-json')
 const packageContents = require('@npmcli/installed-package-contents')

package/lib/debug.js

@@ -18,14 +18,15 @@ const debug = process.env.ARBORIST_DEBUG !== '0' && (
   /\barborist\b/.test(process.env.NODE_DEBUG || '') ||
   process.env.npm_package_name === '@npmcli/arborist' &&
   ['test', 'snap'].includes(process.env.npm_lifecycle_event) ||
-  process.cwd() === require('path').resolve(__dirname, '..')
+  process.cwd() === require('node:path').resolve(__dirname, '..')
 )
 
 module.exports = debug ? fn => fn() : () => {}
 const red = process.stderr.isTTY ? msg => `\x1B[31m${msg}\x1B[39m` : m => m
 module.exports.log = (...msg) => module.exports(() => {
-  const { format } = require('util')
+  const { format } = require('node:util')
   const prefix = `\n${process.pid} ${red(format(msg.shift()))} `
   msg = (prefix + format(...msg).trim().split('\n').join(prefix)).trim()
+  /* eslint-disable-next-line no-console */
   console.error(msg)
 })

package/lib/dep-valid.js

@@ -6,7 +6,7 @@
 
 const semver = require('semver')
 const npa = require('npm-package-arg')
-const { relative } = require('path')
+const { relative } = require('node:path')
 const fromPath = require('./from-path.js')
 
 const depValid = (child, requested, requestor) => {

package/lib/diff.js

@@ -6,7 +6,7 @@
 // for a given branch of the tree being mutated.
 
 const { depth } = require('treeverse')
-const { existsSync } = require('fs')
+const { existsSync } = require('node:fs')
 
 const ssri = require('ssri')
 

package/lib/edge.js

@@ -1,7 +1,7 @@
 // An edge in the dependency graph
 // Represents a dependency relationship of some kind
 
-const util = require('util')
+const util = require('node:util')
 const npa = require('npm-package-arg')
 const depValid = require('./dep-valid.js')
 

package/lib/from-path.js

@@ -3,7 +3,7 @@
 // installed.  directory (ie, symlink) deps also need to be resolved based on
 // their targets, but that's what realpath is
 
-const { dirname } = require('path')
+const { dirname } = require('node:path')
 const npa = require('npm-package-arg')
 
 const fromPath = (node, edge) => {

package/lib/link.js

@@ -2,7 +2,7 @@ const relpath = require('./relpath.js')
 const Node = require('./node.js')
 const _loadDeps = Symbol.for('Arborist.Node._loadDeps')
 const _target = Symbol.for('_target')
-const { dirname } = require('path')
+const { dirname } = require('node:path')
 // defined by Node class
 const _delistFromMeta = Symbol.for('_delistFromMeta')
 const _refreshLocation = Symbol.for('_refreshLocation')

package/lib/node.js

@@ -41,8 +41,8 @@ const gatherDepSet = require('./gather-dep-set.js')
 const treeCheck = require('./tree-check.js')
 const { walkUp } = require('walk-up-path')
 
-const { resolve, relative, dirname, basename } = require('path')
-const util = require('util')
+const { resolve, relative, dirname, basename } = require('node:path')
+const util = require('node:util')
 const _package = Symbol('_package')
 const _parent = Symbol('_parent')
 const _target = Symbol.for('_target')
@@ -119,6 +119,8 @@ class Node {
     // package's dependencies in a virtual root.
     this.sourceReference = sourceReference
 
+    // TODO if this came from pacote.manifest we don't have to do this,
+    // we can be told to skip this step
     const pkg = sourceReference ? sourceReference.package
       : normalize(options.pkg || {})
 

package/lib/packument-cache.js

@@ -0,0 +1,77 @@
+const { LRUCache } = require('lru-cache')
+const { getHeapStatistics } = require('node:v8')
+const { log } = require('proc-log')
+
+// This is an in-memory cache that Pacote uses for packuments.
+// Packuments are usually cached on disk.  This allows for rapid re-requests
+// of the same packument to bypass disk reads.  The tradeoff here is memory
+// usage for disk reads.
+class PackumentCache extends LRUCache {
+  static #heapLimit = Math.floor(getHeapStatistics().heap_size_limit)
+
+  #sizeKey
+  #disposed = new Set()
+
+  #log (...args) {
+    log.silly('packumentCache', ...args)
+  }
+
+  constructor ({
+    // How much of this.#heapLimit to take up
+    heapFactor = 0.25,
+    // How much of this.#maxSize we allow any one packument to take up
+    // Anything over this is not cached
+    maxEntryFactor = 0.5,
+    sizeKey = '_contentLength',
+  } = {}) {
+    const maxSize = Math.floor(PackumentCache.#heapLimit * heapFactor)
+    const maxEntrySize = Math.floor(maxSize * maxEntryFactor)
+    super({
+      maxSize,
+      maxEntrySize,
+      sizeCalculation: (p) => {
+        // Don't cache if we dont know the size
+        // Some versions of pacote set this to `0`, newer versions set it to `null`
+        if (!p[sizeKey]) {
+          return maxEntrySize + 1
+        }
+        if (p[sizeKey] < 10_000) {
+          return p[sizeKey] * 2
+        }
+        if (p[sizeKey] < 1_000_000) {
+          return Math.floor(p[sizeKey] * 1.5)
+        }
+        // It is less beneficial to store a small amount of super large things
+        // at the cost of all other packuments.
+        return maxEntrySize + 1
+      },
+      dispose: (v, k) => {
+        this.#disposed.add(k)
+        this.#log(k, 'dispose')
+      },
+    })
+    this.#sizeKey = sizeKey
+    this.#log(`heap:${PackumentCache.#heapLimit} maxSize:${maxSize} maxEntrySize:${maxEntrySize}`)
+  }
+
+  set (k, v, ...args) {
+    // we use disposed only for a logging signal if we are setting packuments that
+    // have already been evicted from the cache previously. logging here could help
+    // us tune this in the future.
+    const disposed = this.#disposed.has(k)
+    /* istanbul ignore next - this doesnt happen consistently so hard to test without resorting to unit tests  */
+    if (disposed) {
+      this.#disposed.delete(k)
+    }
+    this.#log(k, 'set', `size:${v[this.#sizeKey]} disposed:${disposed}`)
+    return super.set(k, v, ...args)
+  }
+
+  has (k, ...args) {
+    const has = super.has(k, ...args)
+    this.#log(k, `cache-${has ? 'hit' : 'miss'}`)
+    return has
+  }
+}
+
+module.exports = PackumentCache

package/lib/printable.js

@@ -1,7 +1,7 @@
 // helper function to output a clearer visualization
 // of the current node and its descendents
 const localeCompare = require('@isaacs/string-locale-compare')('en')
-const util = require('util')
+const util = require('node:util')
 const relpath = require('./relpath.js')
 
 class ArboristNode {

package/lib/query-selector-all.js

@@ -1,6 +1,6 @@
 'use strict'
 
-const { resolve } = require('path')
+const { resolve } = require('node:path')
 const { parser, arrayDelimiter } = require('@npmcli/query')
 const localeCompare = require('@isaacs/string-locale-compare')('en')
 const { log } = require('proc-log')

package/lib/realpath.js

@@ -5,8 +5,8 @@
 // built-in fs.realpath, because we only care about symbolic links,
 // so we can handle many fewer edge cases.
 
-const { lstat, readlink } = require('fs/promises')
-const { resolve, basename, dirname } = require('path')
+const { lstat, readlink } = require('node:fs/promises')
+const { resolve, basename, dirname } = require('node:path')
 
 const realpathCached = (path, rpcache, stcache, depth) => {
   // just a safety against extremely deep eloops

package/lib/relpath.js

@@ -1,3 +1,3 @@
-const { relative } = require('path')
+const { relative } = require('node:path')
 const relpath = (from, to) => relative(from, to).replace(/\\/g, '/')
 module.exports = relpath

package/lib/retire-path.js

@@ -1,5 +1,5 @@
-const crypto = require('crypto')
-const { dirname, basename, resolve } = require('path')
+const crypto = require('node:crypto')
+const { dirname, basename, resolve } = require('node:path')
 
 // use sha1 because it's faster, and collisions extremely unlikely anyway
 const pathSafeHash = s =>

package/lib/shrinkwrap.js

@@ -42,9 +42,9 @@ const {
   rm,
   stat,
   writeFile,
-} = require('fs/promises')
+} = require('node:fs/promises')
 
-const { resolve, basename, relative } = require('path')
+const { resolve, basename, relative } = require('node:path')
 const specFromLock = require('./spec-from-lock.js')
 const versionFromTgz = require('./version-from-tgz.js')
 const npa = require('npm-package-arg')
@@ -1153,7 +1153,8 @@ class Shrinkwrap {
       && this.originalLockfileVersion !== this.lockfileVersion
     ) {
       log.warn(
-      `Converting lock file (${relative(process.cwd(), this.filename)}) from v${this.originalLockfileVersion} -> v${this.lockfileVersion}`
+        'shrinkwrap',
+        `Converting lock file (${relative(process.cwd(), this.filename)}) from v${this.originalLockfileVersion} -> v${this.lockfileVersion}`
       )
     }
 

package/lib/version-from-tgz.js

@@ -1,6 +1,6 @@
 const semver = require('semver')
-const { basename } = require('path')
-const { URL } = require('url')
+const { basename } = require('node:path')
+const { URL } = require('node:url')
 module.exports = (name, tgz) => {
   const base = basename(tgz)
   if (!base.endsWith('.tgz')) {

package/lib/yarn-lock.js

@@ -29,7 +29,7 @@
 
 const localeCompare = require('@isaacs/string-locale-compare')('en')
 const consistentResolve = require('./consistent-resolve.js')
-const { dirname } = require('path')
+const { dirname } = require('node:path')
 const { breadth } = require('treeverse')
 
 // Sort Yarn entries respecting the yarn.lock sort order

package/package.json

@@ -1,32 +1,33 @@
 {
   "name": "@npmcli/arborist",
-  "version": "7.5.1",
+  "version": "7.5.3",
   "description": "Manage node_modules trees",
   "dependencies": {
     "@isaacs/string-locale-compare": "^1.1.0",
-    "@npmcli/fs": "^3.1.0",
+    "@npmcli/fs": "^3.1.1",
     "@npmcli/installed-package-contents": "^2.1.0",
     "@npmcli/map-workspaces": "^3.0.2",
-    "@npmcli/metavuln-calculator": "^7.1.0",
+    "@npmcli/metavuln-calculator": "^7.1.1",
     "@npmcli/name-from-folder": "^2.0.0",
     "@npmcli/node-gyp": "^3.0.0",
     "@npmcli/package-json": "^5.1.0",
     "@npmcli/query": "^3.1.0",
     "@npmcli/redact": "^2.0.0",
     "@npmcli/run-script": "^8.1.0",
-    "bin-links": "^4.0.1",
-    "cacache": "^18.0.0",
+    "bin-links": "^4.0.4",
+    "cacache": "^18.0.3",
     "common-ancestor-path": "^1.0.1",
-    "hosted-git-info": "^7.0.1",
-    "json-parse-even-better-errors": "^3.0.0",
+    "hosted-git-info": "^7.0.2",
+    "json-parse-even-better-errors": "^3.0.2",
     "json-stringify-nice": "^1.1.4",
+    "lru-cache": "^10.2.2",
     "minimatch": "^9.0.4",
-    "nopt": "^7.0.0",
+    "nopt": "^7.2.1",
     "npm-install-checks": "^6.2.0",
     "npm-package-arg": "^11.0.2",
-    "npm-pick-manifest": "^9.0.0",
-    "npm-registry-fetch": "^17.0.0",
-    "pacote": "^18.0.1",
+    "npm-pick-manifest": "^9.0.1",
+    "npm-registry-fetch": "^17.0.1",
+    "pacote": "^18.0.6",
     "parse-conflict-json": "^3.0.0",
     "proc-log": "^4.2.0",
     "proggy": "^2.0.0",
@@ -34,13 +35,13 @@
     "promise-call-limit": "^3.0.1",
     "read-package-json-fast": "^3.0.2",
     "semver": "^7.3.7",
-    "ssri": "^10.0.5",
+    "ssri": "^10.0.6",
     "treeverse": "^3.0.0",
     "walk-up-path": "^3.0.1"
   },
   "devDependencies": {
     "@npmcli/eslint-config": "^4.0.0",
-    "@npmcli/template-oss": "4.21.3",
+    "@npmcli/template-oss": "4.22.0",
     "benchmark": "^2.1.4",
     "minify-registry-metadata": "^3.0.0",
     "nock": "^13.3.3",
@@ -91,7 +92,7 @@
   },
   "templateOSS": {
     "//@npmcli/template-oss": "This file is partially managed by @npmcli/template-oss. Edits may be overwritten.",
-    "version": "4.21.3",
+    "version": "4.22.0",
     "content": "../../scripts/template-oss/index.js"
   }
 }