@npmcli/arborist 8.0.1 → 8.0.2

package/lib/arborist/isolated-reifier.js

@@ -1,72 +1,105 @@
-const _makeIdealGraph = Symbol('makeIdealGraph')
-const _createIsolatedTree = Symbol.for('createIsolatedTree')
-const _createBundledTree = Symbol('createBundledTree')
 const { mkdirSync } = require('node:fs')
 const pacote = require('pacote')
 const { join } = require('node:path')
 const { depth } = require('treeverse')
 const crypto = require('node:crypto')
 
-// cache complicated function results
-const memoize = (fn) => {
-  const memo = new Map()
-  return async function (arg) {
-    const key = arg
-    if (memo.has(key)) {
-      return memo.get(key)
-    }
-    const result = {}
-    memo.set(key, result)
-    await fn(result, arg)
-    return result
-  }
+// generate short hash key based on the dependency tree starting at this node
+const getKey = (startNode) => {
+  const deps = []
+  const branch = []
+  depth({
+    tree: startNode,
+    getChildren: node => node.dependencies,
+    visit: node => {
+      branch.push(`${node.packageName}@${node.version}`)
+      deps.push(`${branch.join('->')}::${node.resolved}`)
+    },
+    leave: () => {
+      branch.pop()
+    },
+  })
+  deps.sort()
+  // TODO these replaces were originally to deal with node 14 not supporting base64url and likely don't need to happen anymore
+  // Changing this is a pretty significant breaking change, but removing parts of the hash increases collision possibilities (even if slight).
+  const hash = crypto.createHash('shake256', { outputLength: 16 })
+    .update(deps.join(','))
+    .digest('base64')
+    .replace(/\+/g, '-')
+    .replace(/\//g, '_')
+    .replace(/=+$/m, '')
+  return `${startNode.packageName}@${startNode.version}-${hash}`
 }
 
 module.exports = cls => class IsolatedReifier extends cls {
+  #externalProxies = new Map()
+  #processedEdges = new Set()
+  #workspaceProxies = new Map()
+
+  #generateChild (node, location, pkg, inStore, root) {
+    const newChild = {
+      binPaths: [],
+      children: new Map(),
+      edgesIn: new Set(),
+      edgesOut: new Map(),
+      fsChildren: new Set(),
+      /* istanbul ignore next -- emulate Node */
+      getBundler () {
+        return null
+      },
+      global: false,
+      globalTop: false,
+      hasShrinkwrap: false,
+      inDepBundle: false,
+      integrity: null,
+      isInStore: inStore,
+      isLink: false,
+      isProjectRoot: false,
+      isRoot: false,
+      isTop: false,
+      location,
+      name: node.packageName || node.name,
+      optional: node.optional,
+      package: pkg,
+      parent: root,
+      path: join(this.idealGraph.root.localPath, location),
+      realpath: join(this.idealGraph.root.localPath, location),
+      resolved: node.resolved,
+      root,
+      top: { path: this.idealGraph.root.localPath },
+      version: pkg.version,
+    }
+    newChild.target = newChild
+    root.children.set(newChild.location, newChild)
+    root.inventory.set(newChild.location, newChild)
+  }
+
   /**
    * Create an ideal graph.
    *
    * An implementation of npm RFC-0042
    * https://github.com/npm/rfcs/blob/main/accepted/0042-isolated-mode.md
    *
-   * This entire file should be considered technical debt that will be resolved
-   * with an Arborist refactor or rewrite. Embedded logic in Nodes and Links,
-   * and the incremental state of building trees and reifying contains too many
-   * assumptions to do a linked mode properly.
+   * This entire file should be considered technical debt that will be resolved with an Arborist refactor or rewrite.
+   * Embedded logic in Nodes and Links, and the incremental state of building trees and reifying contains too many assumptions to do a linked mode properly.
    *
-   * Instead, this approach takes a tree built from build-ideal-tree, and
-   * returns a new tree-like structure without the embedded logic of Node and
-   * Link classes.
+   * Instead, this approach takes a tree built from build-ideal-tree, and returns a new tree-like structure without the embedded logic of Node and Link classes.
    *
-   * Since the RFC requires leaving the package-lock in place, this approach
-   * temporarily replaces the tree state for a couple of steps of reifying.
+   * Since the RFC requires leaving the package-lock in place, this approach temporarily replaces the tree state for a couple of steps of reifying.
    *
    **/
-  async [_makeIdealGraph] (options) {
-    /* Make sure that the ideal tree is build as the rest of
-     * the algorithm depends on it.
-     */
-    const bitOpt = {
-      ...options,
-      complete: false,
-    }
-    await this.buildIdealTree(bitOpt)
+  async makeIdealGraph () {
     const idealTree = this.idealTree
 
-    this.rootNode = {}
-    const root = this.rootNode
+    this.idealGraph = {
+      external: [],
+      isProjectRoot: true,
+      localLocation: idealTree.location,
+      localPath: idealTree.path,
+    }
     this.counter = 0
 
-    // memoize to cache generating proxy Nodes
-    this.externalProxyMemo = memoize(this.externalProxy.bind(this))
-    this.workspaceProxyMemo = memoize(this.workspaceProxy.bind(this))
-
-    root.external = []
-    root.isProjectRoot = true
-    root.localLocation = idealTree.location
-    root.localPath = idealTree.path
-    root.workspaces = await Promise.all(
-      Array.from(idealTree.fsChildren.values(), this.workspaceProxyMemo))
+    this.idealGraph.workspaces = await Promise.all(Array.from(idealTree.fsChildren.values(), w => this.workspaceProxy(w)))
     const processed = new Set()
     const queue = [idealTree, ...idealTree.fsChildren]
     while (queue.length !== 0) {
@@ -75,42 +108,53 @@ module.exports = cls => class IsolatedReifier extends cls {
         continue
       }
       processed.add(next.location)
-      next.edgesOut.forEach(e => {
-        if (!e.to || (next.package.bundleDependencies || next.package.bundledDependencies || []).includes(e.to.name)) {
-          return
+      next.edgesOut.forEach(edge => {
+        if (edge.to && !(next.package.bundleDependencies || next.package.bundledDependencies || []).includes(edge.to.name)) {
+          queue.push(edge.to)
         }
-        queue.push(e.to)
       })
-      if (!next.isProjectRoot && !next.isWorkspace) {
-        root.external.push(await this.externalProxyMemo(next))
+      // local `file:` deps are in fsChildren but are not workspaces.
+      // they are already handled as workspace-like proxies above and should not go through the external/store extraction path.
+      if (!next.isProjectRoot && !next.isWorkspace && !next.inert && !idealTree.fsChildren.has(next) && !idealTree.fsChildren.has(next.target)) {
+        this.idealGraph.external.push(await this.externalProxy(next))
       }
     }
 
-    await this.assignCommonProperties(idealTree, root)
-
-    this.idealGraph = root
+    await this.assignCommonProperties(idealTree, this.idealGraph)
   }
 
-  async workspaceProxy (result, node) {
+  async workspaceProxy (node) {
+    if (this.#workspaceProxies.has(node)) {
+      return this.#workspaceProxies.get(node)
+    }
+    const result = {}
+    // XXX this goes recursive if we don't set here because assignCommonProperties also calls this.workspaceProxy
+    this.#workspaceProxies.set(node, result)
     result.localLocation = node.location
     result.localPath = node.path
     result.isWorkspace = true
     result.resolved = node.resolved
     await this.assignCommonProperties(node, result)
+    return result
   }
 
-  async externalProxy (result, node) {
-    await this.assignCommonProperties(node, result)
+  async externalProxy (node) {
+    if (this.#externalProxies.has(node)) {
+      return this.#externalProxies.get(node)
+    }
+    const result = {}
+    // XXX this goes recursive if we don't set here because assignCommonProperties also calls this.externalProxy
+    this.#externalProxies.set(node, result)
+    await this.assignCommonProperties(node, result, !node.hasShrinkwrap)
     if (node.hasShrinkwrap) {
       const dir = join(
         node.root.path,
         'node_modules',
         '.store',
-        `${node.name}@${node.version}`
+        `${node.packageName}@${node.version}`
       )
       mkdirSync(dir, { recursive: true })
-      // TODO this approach feels wrong
-      // and shouldn't be necessary for shrinkwraps
+      // TODO this approach feels wrong and shouldn't be necessary for shrinkwraps
       await pacote.extract(node.resolved, dir, {
         ...this.options,
         resolved: node.resolved,
@@ -118,51 +162,81 @@ module.exports = cls => class IsolatedReifier extends cls {
       })
       const Arborist = this.constructor
       const arb = new Arborist({ ...this.options, path: dir })
-      await arb[_makeIdealGraph]({ dev: false })
-      this.rootNode.external.push(...arb.idealGraph.external)
-      arb.idealGraph.external.forEach(e => {
-        e.root = this.rootNode
-        e.id = `${node.id}=>${e.id}`
+      // Make sure that the ideal tree is build as the rest of the algorithm depends on it.
+      await arb.buildIdealTree({
+        complete: false,
+        dev: false,
       })
+      await arb.makeIdealGraph()
+      this.idealGraph.external.push(...arb.idealGraph.external)
+      for (const edge of arb.idealGraph.external) {
+        edge.root = this.idealGraph
+        edge.id = `${node.id}=>${edge.id}`
+      }
       result.localDependencies = []
       result.externalDependencies = arb.idealGraph.externalDependencies
       result.externalOptionalDependencies = arb.idealGraph.externalOptionalDependencies
       result.dependencies = [
         ...result.externalDependencies,
-        ...result.localDependencies,
         ...result.externalOptionalDependencies,
       ]
     }
     result.optional = node.optional
     result.resolved = node.resolved
     result.version = node.version
+    return result
   }
 
-  async assignCommonProperties (node, result) {
-    function validEdgesOut (node) {
-      return [...node.edgesOut.values()].filter(e => e.to && e.to.target && !(node.package.bundledDepenedencies || node.package.bundleDependencies || []).includes(e.to.name))
+  async assignCommonProperties (node, result, populateDeps = true) {
+    result.root = this.idealGraph
+    result.id = this.counter++
+    /* istanbul ignore next - packageName is always set for real packages */
+    result.name = result.isWorkspace ? (node.packageName || node.name) : node.name
+    /* istanbul ignore next - packageName is always set for real packages */
+    result.packageName = node.packageName || node.name
+    result.package = { ...node.package }
+    result.package.bundleDependencies = undefined
+    result.hasInstallScript = node.hasInstallScript
+
+    if (!populateDeps) {
+      return
+    }
+
+    const edges = [...node.edgesOut.values()].filter(edge =>
+      edge.to?.target &&
+      !(node.package.bundledDependencies || node.package.bundleDependencies)?.includes(edge.to.name)
+    )
+    const nonOptionalDeps = edges.filter(edge => !edge.optional).map(edge => edge.to.target)
+
+    // When legacyPeerDeps is enabled, peer dep edges are not created on the node.
+    // Resolve them from the tree so they get symlinked in the store.
+    const peerDeps = node.package.peerDependencies
+    if (peerDeps && node.legacyPeerDeps) {
+      const edgeNames = new Set(edges.map(edge => edge.name))
+      for (const peerName in peerDeps) {
+        if (!edgeNames.has(peerName)) {
+          const resolved = node.resolve(peerName)
+          if (resolved && resolved !== node && !resolved.inert) {
+            nonOptionalDeps.push(resolved.target)
+          }
+        }
+      }
     }
-    const edges = validEdgesOut(node)
-    const optionalDeps = edges.filter(e => e.optional).map(e => e.to.target)
-    const nonOptionalDeps = edges.filter(e => !e.optional).map(e => e.to.target)
 
-    result.localDependencies = await Promise.all(nonOptionalDeps.filter(n => n.isWorkspace).map(this.workspaceProxyMemo))
-    result.externalDependencies = await Promise.all(nonOptionalDeps.filter(n => !n.isWorkspace).map(this.externalProxyMemo))
-    result.externalOptionalDependencies = await Promise.all(optionalDeps.map(this.externalProxyMemo))
+    // local `file:` deps (non-workspace fsChildren) should be treated as local dependencies, not external, so they get symlinked directly instead of being extracted into the store.
+    const isLocal = (n) => n.isWorkspace || node.fsChildren?.has(n)
+    const optionalDeps = edges.filter(edge => edge.optional).map(edge => edge.to.target)
+    result.localDependencies = await Promise.all(nonOptionalDeps.filter(isLocal).map(n => this.workspaceProxy(n)))
+    result.externalDependencies = await Promise.all(nonOptionalDeps.filter(n => !isLocal(n) && !n.inert).map(n => this.externalProxy(n)))
+    result.externalOptionalDependencies = await Promise.all(optionalDeps.filter(n => !n.inert).map(n => this.externalProxy(n)))
     result.dependencies = [
       ...result.externalDependencies,
       ...result.localDependencies,
       ...result.externalOptionalDependencies,
     ]
-    result.root = this.rootNode
-    result.id = this.counter++
-    result.name = node.name
-    result.package = { ...node.package }
-    result.package.bundleDependencies = undefined
-    result.hasInstallScript = node.hasInstallScript
   }
 
-  async [_createBundledTree] () {
+  async #createBundledTree () {
     // TODO: make sure that idealTree object exists
     const idealTree = this.idealTree
     // TODO: test workspaces having bundled deps
@@ -201,253 +275,232 @@ module.exports = cls => class IsolatedReifier extends cls {
       nodes.set(to.location, { location: to.location, resolved: to.resolved, name: to.name, optional: to.optional, pkg: { ...to.package, bundleDependencies: undefined } })
       edges.push({ from: from.isRoot ? 'root' : from.location, to: to.location })
 
-      to.edgesOut.forEach(e => {
+      to.edgesOut.forEach(edge => {
         // an edge out should always have a to
         /* istanbul ignore else */
-        if (e.to) {
-          queue.push({ from: e.from, to: e.to })
+        if (edge.to) {
+          queue.push({ from: edge.from, to: edge.to })
         }
       })
     }
     return { edges, nodes }
   }
 
-  async [_createIsolatedTree] () {
-    await this[_makeIdealGraph](this.options)
-
-    const proxiedIdealTree = this.idealGraph
-
-    const bundledTree = await this[_createBundledTree]()
-
-    const treeHash = (startNode) => {
-      // generate short hash based on the dependency tree
-      // starting at this node
-      const deps = []
-      const branch = []
-      depth({
-        tree: startNode,
-        getChildren: node => node.dependencies,
-        filter: node => node,
-        visit: node => {
-          branch.push(`${node.name}@${node.version}`)
-          deps.push(`${branch.join('->')}::${node.resolved}`)
-        },
-        leave: () => {
-          branch.pop()
-        },
-      })
-      deps.sort()
-      return crypto.createHash('shake256', { outputLength: 16 })
-        .update(deps.join(','))
-        .digest('base64')
-        // Node v14 doesn't support base64url
-        .replace(/\+/g, '-')
-        .replace(/\//g, '_')
-        .replace(/=+$/m, '')
-    }
+  async createIsolatedTree () {
+    await this.makeIdealGraph()
 
-    const getKey = (idealTreeNode) => {
-      return `${idealTreeNode.name}@${idealTreeNode.version}-${treeHash(idealTreeNode)}`
-    }
+    const bundledTree = await this.#createBundledTree()
 
     const root = {
-      fsChildren: [],
-      integrity: null,
-      inventory: new Map(),
-      isLink: false,
-      isRoot: true,
       binPaths: [],
+      children: new Map(),
       edgesIn: new Set(),
       edgesOut: new Map(),
+      fsChildren: new Set(),
+      global: false,
       hasShrinkwrap: false,
+      integrity: null,
+      inventory: new Map(),
+      isLink: false,
+      isProjectRoot: true,
+      isRoot: true,
+      isTop: true,
+      linksIn: new Set(),
+      meta: { loadedFromDisk: false },
+      package: this.idealGraph.root.package,
       parent: null,
+      path: this.idealGraph.root.localPath,
+      realpath: this.idealGraph.root.localPath,
       // TODO: we should probably not reference this.idealTree
       resolved: this.idealTree.resolved,
-      isTop: true,
-      path: proxiedIdealTree.root.localPath,
-      realpath: proxiedIdealTree.root.localPath,
-      package: proxiedIdealTree.root.package,
-      meta: { loadedFromDisk: false },
-      global: false,
-      isProjectRoot: true,
-      children: [],
+      tops: new Set(),
+      workspaces: new Map(),
     }
-    // root.inventory.set('', t)
-    // root.meta = this.idealTree.meta
-    // TODO We should mock better the inventory object because it is used by audit-report.js ... maybe
+    root.inventory.set('', root)
+    root.root = root
+    root.target = root
+    // TODO inventory.query is a stub; audit-report needs 'packageName' support
     root.inventory.query = () => {
       return []
     }
     const processed = new Set()
-    proxiedIdealTree.workspaces.forEach(c => {
+    for (const c of this.idealGraph.workspaces) {
+      const wsName = c.packageName
       const workspace = {
+        binPaths: [],
+        children: new Map(),
         edgesIn: new Set(),
         edgesOut: new Map(),
-        children: [],
+        fsChildren: new Set(),
         hasInstallScript: c.hasInstallScript,
-        binPaths: [],
-        package: c.package,
+        isLink: false,
+        isRoot: false,
+        linksIn: new Set(),
         location: c.localLocation,
+        name: wsName,
+        package: c.package,
         path: c.localPath,
         realpath: c.localPath,
         resolved: c.resolved,
       }
-      root.fsChildren.push(workspace)
+      workspace.target = workspace
+      root.fsChildren.add(workspace)
       root.inventory.set(workspace.location, workspace)
-    })
-    const generateChild = (node, location, pkg, inStore) => {
-      const newChild = {
+
+      // Create workspace Link entry in children for _diffTrees lookup
+      const wsLink = {
+        binPaths: [],
+        children: new Map(),
+        edgesIn: new Set(),
+        edgesOut: new Map(),
+        fsChildren: new Set(),
         global: false,
         globalTop: false,
+        isLink: true,
         isProjectRoot: false,
-        isTop: false,
-        location,
-        name: node.name,
-        optional: node.optional,
-        top: { path: proxiedIdealTree.root.localPath },
-        children: [],
-        edgesIn: new Set(),
-        edgesOut: new Map(),
-        binPaths: [],
-        fsChildren: [],
-        /* istanbul ignore next -- emulate Node */
-        getBundler () {
-          return null
-        },
-        hasShrinkwrap: false,
-        inDepBundle: false,
-        integrity: null,
-        isLink: false,
         isRoot: false,
-        isInStore: inStore,
-        path: join(proxiedIdealTree.root.localPath, location),
-        realpath: join(proxiedIdealTree.root.localPath, location),
-        resolved: node.resolved,
-        version: pkg.version,
-        package: pkg,
+        isTop: false,
+        linksIn: new Set(),
+        location: join('node_modules', wsName),
+        name: wsName,
+        package: workspace.package,
+        parent: root,
+        path: join(root.path, 'node_modules', wsName),
+        realpath: workspace.path,
+        root,
+        target: workspace,
       }
-      newChild.target = newChild
-      root.children.push(newChild)
-      root.inventory.set(newChild.location, newChild)
+      root.children.set(wsLink.name, wsLink)
+      root.inventory.set(wsLink.location, wsLink)
+      root.workspaces.set(wsName, workspace.path)
+      workspace.linksIn.add(wsLink)
     }
-    proxiedIdealTree.external.forEach(c => {
+
+    this.idealGraph.external.forEach(c => {
       const key = getKey(c)
       if (processed.has(key)) {
         return
       }
       processed.add(key)
-      const location = join('node_modules', '.store', key, 'node_modules', c.name)
-      generateChild(c, location, c.package, true)
+      const location = join('node_modules', '.store', key, 'node_modules', c.packageName)
+      this.#generateChild(c, location, c.package, true, root)
     })
+
     bundledTree.nodes.forEach(node => {
-      generateChild(node, node.location, node.pkg, false)
+      this.#generateChild(node, node.location, node.pkg, false, root)
     })
-    bundledTree.edges.forEach(e => {
-      const from = e.from === 'root' ? root : root.inventory.get(e.from)
-      const to = root.inventory.get(e.to)
+
+    bundledTree.edges.forEach(edge => {
+      const from = edge.from === 'root' ? root : root.inventory.get(edge.from)
+      const to = root.inventory.get(edge.to)
       // Maybe optional should be propagated from the original edge
-      const edge = { optional: false, from, to }
-      from.edgesOut.set(to.name, edge)
-      to.edgesIn.add(edge)
+      const newEdge = { optional: false, from, to }
+      from.edgesOut.set(to.name, newEdge)
+      to.edgesIn.add(newEdge)
     })
-    const memo = new Set()
 
-    function processEdges (node, externalEdge) {
-      externalEdge = !!externalEdge
-      const key = getKey(node)
-      if (memo.has(key)) {
-        return
-      }
-      memo.add(key)
-
-      let from, nmFolder
-      if (externalEdge) {
-        const fromLocation = join('node_modules', '.store', key, 'node_modules', node.name)
-        from = root.children.find(c => c.location === fromLocation)
-        nmFolder = join('node_modules', '.store', key, 'node_modules')
-      } else {
-        from = node.isProjectRoot ? root : root.fsChildren.find(c => c.location === node.localLocation)
-        nmFolder = join(node.localLocation, 'node_modules')
-      }
+    this.#processEdges(this.idealGraph, false, root)
+    for (const node of this.idealGraph.workspaces) {
+      this.#processEdges(node, false, root)
+    }
+    return root
+  }
 
-      const processDeps = (dep, optional, external) => {
-        optional = !!optional
-        external = !!external
+  #processEdges (node, externalEdge, root) {
+    const key = getKey(node)
+    if (this.#processedEdges.has(key)) {
+      return
+    }
+    this.#processedEdges.add(key)
 
-        const location = join(nmFolder, dep.name)
-        const binNames = dep.package.bin && Object.keys(dep.package.bin) || []
-        const toKey = getKey(dep)
+    let from, nmFolder
+    if (externalEdge) {
+      const fromLocation = join('node_modules', '.store', key, 'node_modules', node.packageName)
+      from = root.children.get(fromLocation)
+      nmFolder = join('node_modules', '.store', key, 'node_modules')
+    } else {
+      from = node.isProjectRoot ? root : root.inventory.get(node.localLocation)
+      nmFolder = join(node.localLocation, 'node_modules')
+    }
+    /* istanbul ignore next - strict-peer-deps can exclude nodes from the tree */
+    if (!from) {
+      return
+    }
 
-        let target
-        if (external) {
-          const toLocation = join('node_modules', '.store', toKey, 'node_modules', dep.name)
-          target = root.children.find(c => c.location === toLocation)
-        } else {
-          target = root.fsChildren.find(c => c.location === dep.localLocation)
-        }
-        // TODO: we should no-op is an edge has already been created with the same fromKey and toKey
-
-        binNames.forEach(bn => {
-          target.binPaths.push(join(from.realpath, 'node_modules', '.bin', bn))
-        })
-
-        const link = {
-          global: false,
-          globalTop: false,
-          isProjectRoot: false,
-          edgesIn: new Set(),
-          edgesOut: new Map(),
-          binPaths: [],
-          isTop: false,
-          optional,
-          location: location,
-          path: join(dep.root.localPath, nmFolder, dep.name),
-          realpath: target.path,
-          name: toKey,
-          resolved: dep.resolved,
-          top: { path: dep.root.localPath },
-          children: [],
-          fsChildren: [],
-          isLink: true,
-          isStoreLink: true,
-          isRoot: false,
-          package: { _id: 'abc', bundleDependencies: undefined, deprecated: undefined, bin: target.package.bin, scripts: dep.package.scripts },
-          target,
-        }
-        const newEdge1 = { optional, from, to: link }
-        from.edgesOut.set(dep.name, newEdge1)
-        link.edgesIn.add(newEdge1)
-        const newEdge2 = { optional: false, from: link, to: target }
-        link.edgesOut.set(dep.name, newEdge2)
-        target.edgesIn.add(newEdge2)
-        root.children.push(link)
-      }
+    for (const dep of node.localDependencies) {
+      this.#processEdges(dep, false, root)
+      // nonOptional, local
+      this.#processDeps(dep, false, false, root, from, nmFolder)
+    }
+    for (const dep of node.externalDependencies) {
+      this.#processEdges(dep, true, root)
+      // nonOptional, external
+      this.#processDeps(dep, false, true, root, from, nmFolder)
+    }
+    for (const dep of node.externalOptionalDependencies) {
+      this.#processEdges(dep, true, root)
+      // optional, external
+      this.#processDeps(dep, true, true, root, from, nmFolder)
+    }
+  }
 
-      for (const dep of node.localDependencies) {
-        processEdges(dep, false)
-        // nonOptional, local
-        processDeps(dep, false, false)
-      }
-      for (const dep of node.externalDependencies) {
-        processEdges(dep, true)
-        // nonOptional, external
-        processDeps(dep, false, true)
-      }
-      for (const dep of node.externalOptionalDependencies) {
-        processEdges(dep, true)
-        // optional, external
-        processDeps(dep, true, true)
+  #processDeps (dep, optional, external, root, from, nmFolder) {
+    const toKey = getKey(dep)
+
+    let target
+    if (external) {
+      const toLocation = join('node_modules', '.store', toKey, 'node_modules', dep.packageName)
+      target = root.children.get(toLocation)
+    } else {
+      target = root.inventory.get(dep.localLocation)
+    }
+    // TODO: we should no-op is an edge has already been created with the same fromKey and toKey
+    /* istanbul ignore next - strict-peer-deps can exclude nodes from the tree */
+    if (!target) {
+      return
+    }
+
+    if (dep.package.bin) {
+      for (const bn in dep.package.bin) {
+        target.binPaths.push(join(dep.root.localPath, nmFolder, '.bin', bn))
       }
     }
 
-    processEdges(proxiedIdealTree, false)
-    for (const node of proxiedIdealTree.workspaces) {
-      processEdges(node, false)
+    const link = {
+      binPaths: [],
+      children: new Map(),
+      edgesIn: new Set(),
+      edgesOut: new Map(),
+      fsChildren: new Set(),
+      global: false,
+      globalTop: false,
+      isLink: true,
+      isProjectRoot: false,
+      isRoot: false,
+      isStoreLink: true,
+      isTop: false,
+      location: join(nmFolder, dep.name),
+      name: toKey,
+      optional,
+      // TODO _id: 'abc' ?
+      package: { _id: 'abc', bundleDependencies: undefined, deprecated: undefined, bin: target.package.bin, scripts: dep.package.scripts },
+      parent: root,
+      path: join(dep.root.localPath, nmFolder, dep.name),
+      realpath: target.path,
+      resolved: external
+        ? `file:.store/${toKey}/node_modules/${dep.packageName}`
+        : dep.resolved,
+      root,
+      target,
+      version: dep.version,
+      top: { path: dep.root.localPath },
     }
-    root.children.forEach(c => c.parent = root)
-    root.children.forEach(c => c.root = root)
-    root.root = root
-    root.target = root
-    return root
+    const newEdge1 = { optional, from, to: link }
+    from.edgesOut.set(dep.name, newEdge1)
+    link.edgesIn.add(newEdge1)
+    const newEdge2 = { optional: false, from: link, to: target }
+    link.edgesOut.set(dep.name, newEdge2)
+    target.edgesIn.add(newEdge2)
+    root.children.set(link.location, link)
   }
 }

package/lib/arborist/rebuild.js

@@ -297,12 +297,12 @@ module.exports = cls => class Builder extends cls {
         devOptional,
         package: pkg,
         location,
-        isStoreLink,
       } = node.target
 
       // skip any that we know we'll be deleting
-      // or storeLinks
-      if (this[_trashList].has(path) || isStoreLink) {
+      // or links to store entries (their scripts run on the store
+      // entry itself, not through the link)
+      if (this[_trashList].has(path) || (node.isLink && node.target?.isInStore)) {
         return
       }
 
@@ -383,13 +383,21 @@ module.exports = cls => class Builder extends cls {
 
     const timeEnd = time.start(`build:link:${node.location}`)
 
-    const p = binLinks({
+    // On Windows, antivirus/indexer can transiently lock files, causing EPERM/EACCES/EBUSY on the rename inside write-file-atomic (used by bin-links/fix-bin.js), so, retry with backoff.
+    const promiseRetry = require('promise-retry')
+    const p = promiseRetry((retry) => binLinks({
       pkg: node.package,
       path: node.path,
       top: !!(node.isTop || node.globalTop),
       force: this.options.force,
       global: !!node.globalTop,
-    })
+    }).catch(/* istanbul ignore next - Windows-only transient antivirus locks */ err => {
+      if (process.platform === 'win32' &&
+          (err.code === 'EPERM' || err.code === 'EACCES' || err.code === 'EBUSY')) {
+        return retry(err)
+      }
+      throw err
+    }), { retries: 5, minTimeout: 500 })
 
     await (this.#doHandleOptionalFailure
       ? this[_handleOptionalFailure](node, p)

package/lib/arborist/reify.js

@@ -11,11 +11,13 @@ const { log, time } = require('proc-log')
 const hgi = require('hosted-git-info')
 const rpj = require('read-package-json-fast')
 
-const { dirname, resolve, relative, join } = require('node:path')
+const { dirname, resolve, relative, join, sep } = require('node:path')
 const { depth: dfwalk } = require('treeverse')
+const { existsSync } = require('node:fs')
 const {
   lstat,
   mkdir,
+  readdir,
   rm,
   symlink,
 } = require('node:fs/promises')
@@ -77,8 +79,6 @@ const _usePackageLock = Symbol.for('usePackageLock')
 // used by build-ideal-tree mixin
 const _addNodeToTrashList = Symbol.for('addNodeToTrashList')
 
-const _createIsolatedTree = Symbol.for('createIsolatedTree')
-
 module.exports = cls => class Reifier extends cls {
   #bundleMissing = new Set() // child nodes we'd EXPECT to be included in a bundle, but aren't
   #bundleUnpacked = new Set() // the nodes we unpack to read their bundles
@@ -93,6 +93,7 @@ module.exports = cls => class Reifier extends cls {
   #shrinkwrapInflated = new Set()
   #sparseTreeDirs = new Set()
   #sparseTreeRoots = new Set()
+  #linkedActualForDiff = null
 
   constructor (options) {
     super(options)
@@ -136,16 +137,21 @@ module.exports = cls => class Reifier extends cls {
       // this is currently technical debt which will be resolved in a refactor
       // of Node/Link trees
       log.warn('reify', 'The "linked" install strategy is EXPERIMENTAL and may contain bugs.')
-      this.idealTree = await this[_createIsolatedTree]()
+      this.idealTree = await this.createIsolatedTree()
+      this.#linkedActualForDiff = this.#buildLinkedActualForDiff(
+        this.idealTree, this.actualTree
+      )
     }
     await this[_diffTrees]()
     await this[_reifyPackages]()
     if (linked) {
+      await this.#cleanOrphanedStoreEntries()
       // swap back in the idealTree
       // so that the lockfile is preserved
       this.idealTree = oldTree
     }
     await this[_saveIdealTree](options)
+    this.#linkedActualForDiff = null
     // clean up any trash that is still in the tree
     for (const path of this[_trashList]) {
       const loc = relpath(this.idealTree.realpath, path)
@@ -161,7 +167,7 @@ module.exports = cls => class Reifier extends cls {
     // was not changed, delete anything in the ideal and not actual.
     // Then we move the entire idealTree over to this.actualTree, and
     // save the hidden lockfile.
-    if (this.diff && this.diff.filterSet.size) {
+    if (this.diff && this.diff.filterSet.size && !linked) {
       const reroot = new Set()
 
       const { filterSet } = this.diff
@@ -442,9 +448,14 @@ module.exports = cls => class Reifier extends cls {
           if (ideal) {
             filterNodes.push(ideal)
           }
-          const actual = this.actualTree.children.get(ws)
-          if (actual) {
-            filterNodes.push(actual)
+          // Skip actual-side filterNodes when using the linked diff wrapper.
+          // Those nodes have root===actualTree, not root===linkedActualForDiff, and Diff.calculate requires filterNode.root to match actual.
+          // The ideal filterNode alone is sufficient to scope the workspace diff.
+          if (!this.#linkedActualForDiff) {
+            const actual = this.actualTree.children.get(ws)
+            if (actual) {
+              filterNodes.push(actual)
+            }
           }
         }
       }
@@ -465,7 +476,7 @@ module.exports = cls => class Reifier extends cls {
     this.diff = Diff.calculate({
       shrinkwrapInflated: this.#shrinkwrapInflated,
       filterNodes,
-      actual: this.actualTree,
+      actual: this.#linkedActualForDiff || this.actualTree,
       ideal: this.idealTree,
     })
 
@@ -625,6 +636,7 @@ module.exports = cls => class Reifier extends cls {
       // if the directory already exists, made will be undefined. if that's the case
       // we don't want to remove it because we aren't the ones who created it so we
       // omit it from the #sparseTreeRoots
+      /* istanbul ignore next -- pre-existing: mkdir returns undefined when dir exists, covered in reify tests but lost in aggregate coverage merge */
       if (made) {
         this.#sparseTreeRoots.add(made)
       }
@@ -824,6 +836,125 @@ module.exports = cls => class Reifier extends cls {
     }) : p).then(() => node)
   }
 
+  // Build a flat actual tree wrapper for linked installs so the diff can
+  // correctly match store entries that already exist on disk.
+  #buildLinkedActualForDiff (idealTree, actualTree) {
+    const combined = new Map()
+
+    for (const child of actualTree.children.values()) {
+      combined.set(child.path, child)
+    }
+
+    for (const child of idealTree.children.values()) {
+      if (!combined.has(child.path) && (child.isInStore || child.isStoreLink) &&
+          existsSync(child.path)) {
+        const entry = {
+          global: false,
+          globalTop: false,
+          isProjectRoot: false,
+          isTop: false,
+          location: child.location,
+          name: child.name,
+          optional: child.optional,
+          top: child.top,
+          children: [],
+          edgesIn: new Set(),
+          edgesOut: new Map(),
+          binPaths: [],
+          fsChildren: [],
+          /* istanbul ignore next -- emulate Node */
+          getBundler () {
+            return null
+          },
+          hasShrinkwrap: false,
+          inDepBundle: false,
+          integrity: null,
+          isLink: Boolean(child.isLink),
+          isRoot: false,
+          isInStore: Boolean(child.isInStore),
+          path: child.path,
+          realpath: child.realpath,
+          resolved: child.resolved,
+          version: child.version,
+          package: child.package,
+        }
+        entry.target = entry
+        if (child.isLink && combined.has(child.realpath)) {
+          entry.target = combined.get(child.realpath)
+        }
+        combined.set(child.path, entry)
+      }
+    }
+
+    const origGet = actualTree.children.get.bind(actualTree.children)
+    const combinedGet = combined.get.bind(combined)
+    /* istanbul ignore next -- only reached during scoped workspace installs */
+    combined.get = (key) => combinedGet(key) || origGet(key)
+
+    const wrapper = {
+      isRoot: true,
+      isLink: actualTree.isLink,
+      target: actualTree.target,
+      fsChildren: actualTree.fsChildren,
+      path: actualTree.path,
+      realpath: actualTree.realpath,
+      edgesOut: actualTree.edgesOut,
+      inventory: actualTree.inventory,
+      package: actualTree.package,
+      resolved: actualTree.resolved,
+      version: actualTree.version,
+      integrity: actualTree.integrity,
+      binPaths: actualTree.binPaths,
+      hasShrinkwrap: false,
+      inDepBundle: false,
+      parent: null,
+      children: combined,
+    }
+
+    for (const child of combined.values()) {
+      if (!child.parent) {
+        child.parent = wrapper
+        child.root = wrapper
+      }
+    }
+
+    return wrapper
+  }
+
+  // After a linked install, scan node_modules/.store/ and remove any
+  // directories that are not referenced by the current ideal tree.
+  async #cleanOrphanedStoreEntries () {
+    const storeDir = resolve(this.path, 'node_modules', '.store')
+    let entries
+    try {
+      entries = await readdir(storeDir)
+    } catch {
+      return
+    }
+
+    const validKeys = new Set()
+    for (const child of this.idealTree.children.values()) {
+      if (child.isInStore) {
+        const key = child.location.split(sep)[2]
+        validKeys.add(key)
+      }
+    }
+
+    const orphaned = entries.filter(e => !validKeys.has(e))
+    if (!orphaned.length) {
+      return
+    }
+
+    log.silly('reify', 'cleaning orphaned store entries', orphaned)
+    await promiseAllRejectLate(
+      orphaned.map(e =>
+        rm(resolve(storeDir, e), { recursive: true, force: true })
+          .catch(/* istanbul ignore next -- rm with force rarely fails */
+            er => log.warn('cleanup', `Failed to remove orphaned store entry ${e}`, er))
+      )
+    )
+  }
+
   #registryResolved (resolved) {
     // the default registry url is a magic value meaning "the currently
     // configured registry".

package/lib/diff.js

@@ -69,6 +69,7 @@ class Diff {
         tree: filterNode,
         visit: node => filterSet.add(node),
         getChildren: node => {
+          const orig = node
           node = node.target
           const loc = node.location
           const idealNode = ideal.inventory.get(loc)
@@ -85,7 +86,12 @@ class Diff {
             }
           }
 
-          return ideals.concat(actuals)
+          const result = ideals.concat(actuals)
+          // Include link targets so store entries end up in filterSet
+          if (orig.isLink) {
+            result.push(node)
+          }
+          return result
         },
       })
     }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@npmcli/arborist",
-  "version": "8.0.1",
+  "version": "8.0.2",
   "description": "Manage node_modules trees",
   "dependencies": {
     "@isaacs/string-locale-compare": "^1.1.0",
@@ -33,6 +33,7 @@
     "proggy": "^3.0.0",
     "promise-all-reject-late": "^1.0.0",
     "promise-call-limit": "^3.0.1",
+    "promise-retry": "^2.0.1",
     "read-package-json-fast": "^4.0.0",
     "semver": "^7.3.7",
     "ssri": "^12.0.0",
@@ -41,7 +42,7 @@
   },
   "devDependencies": {
     "@npmcli/eslint-config": "^5.0.1",
-    "@npmcli/template-oss": "4.24.4",
+    "@npmcli/template-oss": "4.29.0",
     "benchmark": "^2.1.4",
     "minify-registry-metadata": "^4.0.0",
     "nock": "^13.3.3",
@@ -93,7 +94,7 @@
   },
   "templateOSS": {
     "//@npmcli/template-oss": "This file is partially managed by @npmcli/template-oss. Edits may be overwritten.",
-    "version": "4.24.4",
+    "version": "4.29.0",
     "content": "../../scripts/template-oss/index.js"
   }
 }