@npmcli/arborist 7.5.1 → 7.5.2

package/lib/arborist/build-ideal-tree.js

@@ -53,48 +53,26 @@ const _addNodeToTrashList = Symbol.for('addNodeToTrashList')
 // they'll affect things deeper in, then alphabetical for consistency between
 // installs
 class DepsQueue {
-  // [{ sorted, items }] indexed by depth
   #deps = []
   #sorted = true
-  #minDepth = 0
-  #length = 0
 
   get length () {
-    return this.#length
+    return this.#deps.length
   }
 
   push (item) {
-    if (!this.#deps[item.depth]) {
-      this.#length++
-      this.#deps[item.depth] = { sorted: true, items: [item] }
-      // no minDepth check needed, this branch is only reached when we are in
-      // the middle of a shallower depth and creating a new one
-      return
-    }
-    if (!this.#deps[item.depth].items.includes(item)) {
-      this.#length++
-      this.#deps[item.depth].sorted = false
-      this.#deps[item.depth].items.push(item)
-      if (item.depth < this.#minDepth) {
-        this.#minDepth = item.depth
-      }
+    if (!this.#deps.includes(item)) {
+      this.#sorted = false
+      this.#deps.push(item)
     }
   }
 
   pop () {
-    let depth
-    while (!depth?.items.length) {
-      depth = this.#deps[this.#minDepth]
-      if (!depth?.items.length) {
-        this.#minDepth++
-      }
+    if (!this.#sorted) {
+      this.#deps.sort((a, b) => (a.depth - b.depth) || localeCompare(a.path, b.path))
+      this.#sorted = true
     }
-    if (!depth.sorted) {
-      depth.items.sort((a, b) => localeCompare(a.path, b.path))
-      depth.sorted = true
-    }
-    this.#length--
-    return depth.items.shift()
+    return this.#deps.shift()
   }
 }
 
@@ -1022,6 +1000,10 @@ This is a one-time fix-up, please be patient...
           // may well be an optional dep that has gone missing.  it'll
           // fail later anyway.
           for (const e of this.#problemEdges(placed)) {
+            // XXX This is somehow load bearing.  This makes tests that print
+            // the ideal tree of a tree with tarball dependencies fail. This
+            // can't be changed or removed till we figure out why
+            // The test is named "tarball deps with transitive tarball deps"
             promises.push(() =>
               this.#fetchManifest(npa.resolve(e.name, e.spec, fromPath(placed, e)))
                 .catch(() => null)
@@ -1204,6 +1186,7 @@ This is a one-time fix-up, please be patient...
     const options = {
       ...this.options,
       avoid: this.#avoidRange(spec.name),
+      fullMetadata: true,
     }
     // get the intended spec and stored metadata from yarn.lock file,
     // if available and valid.
@@ -1212,19 +1195,10 @@ This is a one-time fix-up, please be patient...
     if (this.#manifests.has(spec.raw)) {
       return this.#manifests.get(spec.raw)
     } else {
-      const cleanRawSpec = redact(spec.rawSpec)
-      log.silly('fetch manifest', spec.raw.replace(spec.rawSpec, cleanRawSpec))
-      const o = {
-        ...options,
-        fullMetadata: true,
-      }
-      const p = pacote.manifest(spec, o)
-        .then(({ license, ...mani }) => {
-          this.#manifests.set(spec.raw, mani)
-          return mani
-        })
-      this.#manifests.set(spec.raw, p)
-      return p
+      log.silly('fetch manifest', spec.raw.replace(spec.rawSpec, redact(spec.rawSpec)))
+      const mani = await pacote.manifest(spec, options)
+      this.#manifests.set(spec.raw, mani)
+      return mani
     }
   }
 

package/lib/arborist/index.js

@@ -31,10 +31,10 @@ const { homedir } = require('os')
 const { depth } = require('treeverse')
 const mapWorkspaces = require('@npmcli/map-workspaces')
 const { log, time } = require('proc-log')
-
 const { saveTypeMap } = require('../add-rm-pkg-deps.js')
 const AuditReport = require('../audit-report.js')
 const relpath = require('../relpath.js')
+const PackumentCache = require('../packument-cache.js')
 
 const mixins = [
   require('../tracker.js'),
@@ -82,7 +82,7 @@ class Arborist extends Base {
       installStrategy: options.global ? 'shallow' : (options.installStrategy ? options.installStrategy : 'hoisted'),
       lockfileVersion: lockfileVersion(options.lockfileVersion),
       packageLockOnly: !!options.packageLockOnly,
-      packumentCache: options.packumentCache || new Map(),
+      packumentCache: options.packumentCache || new PackumentCache(),
       path: options.path || '.',
       rebuildBundle: 'rebuildBundle' in options ? !!options.rebuildBundle : true,
       replaceRegistryHost: options.replaceRegistryHost,

package/lib/node.js

@@ -119,6 +119,8 @@ class Node {
     // package's dependencies in a virtual root.
     this.sourceReference = sourceReference
 
+    // TODO if this came from pacote.manifest we don't have to do this,
+    // we can be told to skip this step
     const pkg = sourceReference ? sourceReference.package
       : normalize(options.pkg || {})
 

package/lib/packument-cache.js

@@ -0,0 +1,77 @@
+const { LRUCache } = require('lru-cache')
+const { getHeapStatistics } = require('node:v8')
+const { log } = require('proc-log')
+
+// This is an in-memory cache that Pacote uses for packuments.
+// Packuments are usually cached on disk.  This allows for rapid re-requests
+// of the same packument to bypass disk reads.  The tradeoff here is memory
+// usage for disk reads.
+class PackumentCache extends LRUCache {
+  static #heapLimit = Math.floor(getHeapStatistics().heap_size_limit)
+
+  #sizeKey
+  #disposed = new Set()
+
+  #log (...args) {
+    log.silly('packumentCache', ...args)
+  }
+
+  constructor ({
+    // How much of this.#heapLimit to take up
+    heapFactor = 0.25,
+    // How much of this.#maxSize we allow any one packument to take up
+    // Anything over this is not cached
+    maxEntryFactor = 0.5,
+    sizeKey = '_contentLength',
+  } = {}) {
+    const maxSize = Math.floor(PackumentCache.#heapLimit * heapFactor)
+    const maxEntrySize = Math.floor(maxSize * maxEntryFactor)
+    super({
+      maxSize,
+      maxEntrySize,
+      sizeCalculation: (p) => {
+        // Don't cache if we dont know the size
+        // Some versions of pacote set this to `0`, newer versions set it to `null`
+        if (!p[sizeKey]) {
+          return maxEntrySize + 1
+        }
+        if (p[sizeKey] < 10_000) {
+          return p[sizeKey] * 2
+        }
+        if (p[sizeKey] < 1_000_000) {
+          return Math.floor(p[sizeKey] * 1.5)
+        }
+        // It is less beneficial to store a small amount of super large things
+        // at the cost of all other packuments.
+        return maxEntrySize + 1
+      },
+      dispose: (v, k) => {
+        this.#disposed.add(k)
+        this.#log(k, 'dispose')
+      },
+    })
+    this.#sizeKey = sizeKey
+    this.#log(`heap:${PackumentCache.#heapLimit} maxSize:${maxSize} maxEntrySize:${maxEntrySize}`)
+  }
+
+  set (k, v, ...args) {
+    // we use disposed only for a logging signal if we are setting packuments that
+    // have already been evicted from the cache previously. logging here could help
+    // us tune this in the future.
+    const disposed = this.#disposed.has(k)
+    /* istanbul ignore next - this doesnt happen consistently so hard to test without resorting to unit tests  */
+    if (disposed) {
+      this.#disposed.delete(k)
+    }
+    this.#log(k, 'set', `size:${v[this.#sizeKey]} disposed:${disposed}`)
+    return super.set(k, v, ...args)
+  }
+
+  has (k, ...args) {
+    const has = super.has(k, ...args)
+    this.#log(k, `cache-${has ? 'hit' : 'miss'}`)
+    return has
+  }
+}
+
+module.exports = PackumentCache

package/lib/shrinkwrap.js

@@ -1153,7 +1153,8 @@ class Shrinkwrap {
       && this.originalLockfileVersion !== this.lockfileVersion
     ) {
       log.warn(
-      `Converting lock file (${relative(process.cwd(), this.filename)}) from v${this.originalLockfileVersion} -> v${this.lockfileVersion}`
+        'shrinkwrap',
+        `Converting lock file (${relative(process.cwd(), this.filename)}) from v${this.originalLockfileVersion} -> v${this.lockfileVersion}`
       )
     }
 

package/package.json

@@ -1,32 +1,33 @@
 {
   "name": "@npmcli/arborist",
-  "version": "7.5.1",
+  "version": "7.5.2",
   "description": "Manage node_modules trees",
   "dependencies": {
     "@isaacs/string-locale-compare": "^1.1.0",
-    "@npmcli/fs": "^3.1.0",
+    "@npmcli/fs": "^3.1.1",
     "@npmcli/installed-package-contents": "^2.1.0",
     "@npmcli/map-workspaces": "^3.0.2",
-    "@npmcli/metavuln-calculator": "^7.1.0",
+    "@npmcli/metavuln-calculator": "^7.1.1",
     "@npmcli/name-from-folder": "^2.0.0",
     "@npmcli/node-gyp": "^3.0.0",
     "@npmcli/package-json": "^5.1.0",
     "@npmcli/query": "^3.1.0",
     "@npmcli/redact": "^2.0.0",
     "@npmcli/run-script": "^8.1.0",
-    "bin-links": "^4.0.1",
-    "cacache": "^18.0.0",
+    "bin-links": "^4.0.4",
+    "cacache": "^18.0.3",
     "common-ancestor-path": "^1.0.1",
-    "hosted-git-info": "^7.0.1",
-    "json-parse-even-better-errors": "^3.0.0",
+    "hosted-git-info": "^7.0.2",
+    "json-parse-even-better-errors": "^3.0.2",
     "json-stringify-nice": "^1.1.4",
+    "lru-cache": "^10.2.2",
     "minimatch": "^9.0.4",
-    "nopt": "^7.0.0",
+    "nopt": "^7.2.1",
     "npm-install-checks": "^6.2.0",
     "npm-package-arg": "^11.0.2",
-    "npm-pick-manifest": "^9.0.0",
-    "npm-registry-fetch": "^17.0.0",
-    "pacote": "^18.0.1",
+    "npm-pick-manifest": "^9.0.1",
+    "npm-registry-fetch": "^17.0.1",
+    "pacote": "^18.0.6",
     "parse-conflict-json": "^3.0.0",
     "proc-log": "^4.2.0",
     "proggy": "^2.0.0",
@@ -34,13 +35,13 @@
     "promise-call-limit": "^3.0.1",
     "read-package-json-fast": "^3.0.2",
     "semver": "^7.3.7",
-    "ssri": "^10.0.5",
+    "ssri": "^10.0.6",
     "treeverse": "^3.0.0",
     "walk-up-path": "^3.0.1"
   },
   "devDependencies": {
     "@npmcli/eslint-config": "^4.0.0",
-    "@npmcli/template-oss": "4.21.3",
+    "@npmcli/template-oss": "4.22.0",
     "benchmark": "^2.1.4",
     "minify-registry-metadata": "^3.0.0",
     "nock": "^13.3.3",
@@ -91,7 +92,7 @@
   },
   "templateOSS": {
     "//@npmcli/template-oss": "This file is partially managed by @npmcli/template-oss. Edits may be overwritten.",
-    "version": "4.21.3",
+    "version": "4.22.0",
     "content": "../../scripts/template-oss/index.js"
   }
 }