@npmcli/arborist 6.1.5 → 6.2.0

package/lib/arborist/build-ideal-tree.js

@@ -12,6 +12,7 @@ const { readdirScoped } = require('@npmcli/fs')
 const { lstat, readlink } = require('fs/promises')
 const { depth } = require('treeverse')
 const log = require('proc-log')
+const { cleanUrl } = require('npm-registry-fetch')
 
 const {
   OK,
@@ -1210,7 +1211,8 @@ This is a one-time fix-up, please be patient...
     if (this[_manifests].has(spec.raw)) {
       return this[_manifests].get(spec.raw)
     } else {
-      log.silly('fetch manifest', spec.raw)
+      const cleanRawSpec = cleanUrl(spec.rawSpec)
+      log.silly('fetch manifest', spec.raw.replace(spec.rawSpec, cleanRawSpec))
       const p = pacote.manifest(spec, options)
         .then(mani => {
           this[_manifests].set(spec.raw, mani)
@@ -1230,6 +1232,7 @@ This is a one-time fix-up, please be patient...
     const isWorkspace = this.idealTree.workspaces && this.idealTree.workspaces.has(spec.name)
 
     // spec is a directory, link it unless installLinks is set or it's a workspace
+    // TODO post arborist refactor, will need to check for installStrategy=linked
     if (spec.type === 'directory' && (isWorkspace || !installLinks)) {
       return this[_linkFromSpec](name, spec, parent, edge)
     }

package/lib/arborist/index.js

@@ -42,6 +42,7 @@ const mixins = [
   require('./load-virtual.js'),
   require('./rebuild.js'),
   require('./reify.js'),
+  require('./isolated-reifier.js'),
 ]
 
 const _workspacesEnabled = Symbol.for('workspacesEnabled')

package/lib/arborist/isolated-reifier.js

@@ -0,0 +1,453 @@
+const _makeIdealGraph = Symbol('makeIdealGraph')
+const _createIsolatedTree = Symbol.for('createIsolatedTree')
+const _createBundledTree = Symbol('createBundledTree')
+const fs = require('fs')
+const pacote = require('pacote')
+const { join } = require('path')
+const { depth } = require('treeverse')
+const crypto = require('crypto')
+
+// cache complicated function results
+const memoize = (fn) => {
+  const memo = new Map()
+  return async function (arg) {
+    const key = arg
+    if (memo.has(key)) {
+      return memo.get(key)
+    }
+    const result = {}
+    memo.set(key, result)
+    await fn(result, arg)
+    return result
+  }
+}
+
+module.exports = cls => class IsolatedReifier extends cls {
+  /**
+   * Create an ideal graph.
+   *
+   * An implementation of npm RFC-0042
+   * https://github.com/npm/rfcs/blob/main/accepted/0042-isolated-mode.md
+   *
+   * This entire file should be considered technical debt that will be resolved
+   * with an Arborist refactor or rewrite. Embedded logic in Nodes and Links,
+   * and the incremental state of building trees and reifying contains too many
+   * assumptions to do a linked mode properly.
+   *
+   * Instead, this approach takes a tree built from build-ideal-tree, and
+   * returns a new tree-like structure without the embedded logic of Node and
+   * Link classes.
+   *
+   * Since the RFC requires leaving the package-lock in place, this approach
+   * temporarily replaces the tree state for a couple of steps of reifying.
+   *
+   **/
+  async [_makeIdealGraph] (options) {
+    /* Make sure that the ideal tree is build as the rest of
+     * the algorithm depends on it.
+     */
+    const bitOpt = {
+      ...options,
+      complete: false,
+    }
+    await this.buildIdealTree(bitOpt)
+    const idealTree = this.idealTree
+
+    this.rootNode = {}
+    const root = this.rootNode
+    this.counter = 0
+
+    // memoize to cache generating proxy Nodes
+    this.externalProxyMemo = memoize(this.externalProxy.bind(this))
+    this.workspaceProxyMemo = memoize(this.workspaceProxy.bind(this))
+
+    root.external = []
+    root.isProjectRoot = true
+    root.localLocation = idealTree.location
+    root.localPath = idealTree.path
+    root.workspaces = await Promise.all(
+      Array.from(idealTree.fsChildren.values(), this.workspaceProxyMemo))
+    const processed = new Set()
+    const queue = [idealTree, ...idealTree.fsChildren]
+    while (queue.length !== 0) {
+      const next = queue.pop()
+      if (processed.has(next.location)) {
+        continue
+      }
+      processed.add(next.location)
+      next.edgesOut.forEach(e => {
+        if (!e.to || (next.package.bundleDependencies || next.package.bundledDependencies || []).includes(e.to.name)) {
+          return
+        }
+        queue.push(e.to)
+      })
+      if (!next.isProjectRoot && !next.isWorkspace) {
+        root.external.push(await this.externalProxyMemo(next))
+      }
+    }
+
+    await this.assignCommonProperties(idealTree, root)
+
+    this.idealGraph = root
+  }
+
+  async workspaceProxy (result, node) {
+    result.localLocation = node.location
+    result.localPath = node.path
+    result.isWorkspace = true
+    result.resolved = node.resolved
+    await this.assignCommonProperties(node, result)
+  }
+
+  async externalProxy (result, node) {
+    await this.assignCommonProperties(node, result)
+    if (node.hasShrinkwrap) {
+      const dir = join(
+        node.root.path,
+        'node_modules',
+        '.store',
+        `${node.name}@${node.version}`
+      )
+      fs.mkdirSync(dir, { recursive: true })
+      // TODO this approach feels wrong
+      // and shouldn't be necessary for shrinkwraps
+      await pacote.extract(node.resolved, dir, {
+        ...this.options,
+        resolved: node.resolved,
+        integrity: node.integrity,
+      })
+      const Arborist = this.constructor
+      const arb = new Arborist({ ...this.options, path: dir })
+      await arb[_makeIdealGraph]({ dev: false })
+      this.rootNode.external.push(...arb.idealGraph.external)
+      arb.idealGraph.external.forEach(e => {
+        e.root = this.rootNode
+        e.id = `${node.id}=>${e.id}`
+      })
+      result.localDependencies = []
+      result.externalDependencies = arb.idealGraph.externalDependencies
+      result.externalOptionalDependencies = arb.idealGraph.externalOptionalDependencies
+      result.dependencies = [
+        ...result.externalDependencies,
+        ...result.localDependencies,
+        ...result.externalOptionalDependencies,
+      ]
+    }
+    result.optional = node.optional
+    result.resolved = node.resolved
+    result.version = node.version
+  }
+
+  async assignCommonProperties (node, result) {
+    function validEdgesOut (node) {
+      return [...node.edgesOut.values()].filter(e => e.to && e.to.target && !(node.package.bundledDepenedencies || node.package.bundleDependencies || []).includes(e.to.name))
+    }
+    const edges = validEdgesOut(node)
+    const optionalDeps = edges.filter(e => e.optional).map(e => e.to.target)
+    const nonOptionalDeps = edges.filter(e => !e.optional).map(e => e.to.target)
+
+    result.localDependencies = await Promise.all(nonOptionalDeps.filter(n => n.isWorkspace).map(this.workspaceProxyMemo))
+    result.externalDependencies = await Promise.all(nonOptionalDeps.filter(n => !n.isWorkspace).map(this.externalProxyMemo))
+    result.externalOptionalDependencies = await Promise.all(optionalDeps.map(this.externalProxyMemo))
+    result.dependencies = [
+      ...result.externalDependencies,
+      ...result.localDependencies,
+      ...result.externalOptionalDependencies,
+    ]
+    result.root = this.rootNode
+    result.id = this.counter++
+    result.name = node.name
+    result.package = { ...node.package }
+    result.package.bundleDependencies = undefined
+    result.hasInstallScript = node.hasInstallScript
+  }
+
+  async [_createBundledTree] () {
+    // TODO: make sure that idealTree object exists
+    const idealTree = this.idealTree
+    // TODO: test workspaces having bundled deps
+    const queue = []
+
+    for (const [, edge] of idealTree.edgesOut) {
+      if (edge.to && (idealTree.package.bundleDependencies || idealTree.package.bundledDependencies || []).includes(edge.to.name)) {
+        queue.push({ from: idealTree, to: edge.to })
+      }
+    }
+    for (const child of idealTree.fsChildren) {
+      for (const [, edge] of child.edgesOut) {
+        if (edge.to && (child.package.bundleDependencies || child.package.bundledDependencies || []).includes(edge.to.name)) {
+          queue.push({ from: child, to: edge.to })
+        }
+      }
+    }
+
+    const processed = new Set()
+    const nodes = new Map()
+    const edges = []
+    while (queue.length !== 0) {
+      const nextEdge = queue.pop()
+      const key = `${nextEdge.from.location}=>${nextEdge.to.location}`
+      // should be impossible, unless bundled is duped
+      /* istanbul ignore next */
+      if (processed.has(key)) {
+        continue
+      }
+      processed.add(key)
+      const from = nextEdge.from
+      if (!from.isRoot && !from.isWorkspace) {
+        nodes.set(from.location, { location: from.location, resolved: from.resolved, name: from.name, optional: from.optional, pkg: { ...from.package, bundleDependencies: undefined } })
+      }
+      const to = nextEdge.to
+      nodes.set(to.location, { location: to.location, resolved: to.resolved, name: to.name, optional: to.optional, pkg: { ...to.package, bundleDependencies: undefined } })
+      edges.push({ from: from.isRoot ? 'root' : from.location, to: to.location })
+
+      to.edgesOut.forEach(e => {
+        // an edge out should always have a to
+        /* istanbul ignore else */
+        if (e.to) {
+          queue.push({ from: e.from, to: e.to })
+        }
+      })
+    }
+    return { edges, nodes }
+  }
+
+  async [_createIsolatedTree] (idealTree) {
+    await this[_makeIdealGraph](this.options)
+
+    const proxiedIdealTree = this.idealGraph
+
+    const bundledTree = await this[_createBundledTree]()
+
+    const treeHash = (startNode) => {
+      // generate short hash based on the dependency tree
+      // starting at this node
+      const deps = []
+      const branch = []
+      depth({
+        tree: startNode,
+        getChildren: node => node.dependencies,
+        filter: node => node,
+        visit: node => {
+          branch.push(`${node.name}@${node.version}`)
+          deps.push(`${branch.join('->')}::${node.resolved}`)
+        },
+        leave: () => {
+          branch.pop()
+        },
+      })
+      deps.sort()
+      return crypto.createHash('shake256', { outputLength: 16 })
+        .update(deps.join(','))
+        .digest('base64')
+        // Node v14 doesn't support base64url
+        .replace(/\+/g, '-')
+        .replace(/\//g, '_')
+        .replace(/=+$/m, '')
+    }
+
+    const getKey = (idealTreeNode) => {
+      return `${idealTreeNode.name}@${idealTreeNode.version}-${treeHash(idealTreeNode)}`
+    }
+
+    const root = {
+      fsChildren: [],
+      integrity: null,
+      inventory: new Map(),
+      isLink: false,
+      isRoot: true,
+      binPaths: [],
+      edgesIn: new Set(),
+      edgesOut: new Map(),
+      hasShrinkwrap: false,
+      parent: null,
+      // TODO: we should probably not reference this.idealTree
+      resolved: this.idealTree.resolved,
+      isTop: true,
+      path: proxiedIdealTree.root.localPath,
+      realpath: proxiedIdealTree.root.localPath,
+      package: proxiedIdealTree.root.package,
+      meta: { loadedFromDisk: false },
+      global: false,
+      isProjectRoot: true,
+      children: [],
+    }
+    // root.inventory.set('', t)
+    // root.meta = this.idealTree.meta
+    // TODO We should mock better the inventory object because it is used by audit-report.js ... maybe
+    root.inventory.query = () => {
+      return []
+    }
+    const processed = new Set()
+    proxiedIdealTree.workspaces.forEach(c => {
+      const workspace = {
+        edgesIn: new Set(),
+        edgesOut: new Map(),
+        children: [],
+        hasInstallScript: c.hasInstallScript,
+        binPaths: [],
+        package: c.package,
+        location: c.localLocation,
+        path: c.localPath,
+        realpath: c.localPath,
+        resolved: c.resolved,
+      }
+      root.fsChildren.push(workspace)
+      root.inventory.set(workspace.location, workspace)
+    })
+    const generateChild = (node, location, pkg, inStore) => {
+      const newChild = {
+        global: false,
+        globalTop: false,
+        isProjectRoot: false,
+        isTop: false,
+        location,
+        name: node.name,
+        optional: node.optional,
+        top: { path: proxiedIdealTree.root.localPath },
+        children: [],
+        edgesIn: new Set(),
+        edgesOut: new Map(),
+        binPaths: [],
+        fsChildren: [],
+        /* istanbul ignore next -- emulate Node */
+        getBundler () {
+          return null
+        },
+        hasShrinkwrap: false,
+        inDepBundle: false,
+        integrity: null,
+        isLink: false,
+        isRoot: false,
+        isInStore: inStore,
+        path: join(proxiedIdealTree.root.localPath, location),
+        realpath: join(proxiedIdealTree.root.localPath, location),
+        resolved: node.resolved,
+        version: pkg.version,
+        package: pkg,
+      }
+      newChild.target = newChild
+      root.children.push(newChild)
+      root.inventory.set(newChild.location, newChild)
+    }
+    proxiedIdealTree.external.forEach(c => {
+      const key = getKey(c)
+      if (processed.has(key)) {
+        return
+      }
+      processed.add(key)
+      const location = join('node_modules', '.store', key, 'node_modules', c.name)
+      generateChild(c, location, c.package, true)
+    })
+    bundledTree.nodes.forEach(node => {
+      generateChild(node, node.location, node.pkg, false)
+    })
+    bundledTree.edges.forEach(e => {
+      const from = e.from === 'root' ? root : root.inventory.get(e.from)
+      const to = root.inventory.get(e.to)
+      // Maybe optional should be propagated from the original edge
+      const edge = { optional: false, from, to }
+      from.edgesOut.set(to.name, edge)
+      to.edgesIn.add(edge)
+    })
+    const memo = new Set()
+
+    function processEdges (node, externalEdge) {
+      externalEdge = !!externalEdge
+      const key = getKey(node)
+      if (memo.has(key)) {
+        return
+      }
+      memo.add(key)
+
+      let from, nmFolder
+      if (externalEdge) {
+        const fromLocation = join('node_modules', '.store', key, 'node_modules', node.name)
+        from = root.children.find(c => c.location === fromLocation)
+        nmFolder = join('node_modules', '.store', key, 'node_modules')
+      } else {
+        from = node.isProjectRoot ? root : root.fsChildren.find(c => c.location === node.localLocation)
+        nmFolder = join(node.localLocation, 'node_modules')
+      }
+
+      const processDeps = (dep, optional, external) => {
+        optional = !!optional
+        external = !!external
+
+        const location = join(nmFolder, dep.name)
+        const binNames = dep.package.bin && Object.keys(dep.package.bin) || []
+        const toKey = getKey(dep)
+
+        let target
+        if (external) {
+          const toLocation = join('node_modules', '.store', toKey, 'node_modules', dep.name)
+          target = root.children.find(c => c.location === toLocation)
+        } else {
+          target = root.fsChildren.find(c => c.location === dep.localLocation)
+        }
+        // TODO: we should no-op is an edge has already been created with the same fromKey and toKey
+
+        binNames.forEach(bn => {
+          target.binPaths.push(join(from.realpath, 'node_modules', '.bin', bn))
+        })
+
+        const link = {
+          global: false,
+          globalTop: false,
+          isProjectRoot: false,
+          edgesIn: new Set(),
+          edgesOut: new Map(),
+          binPaths: [],
+          isTop: false,
+          optional,
+          location: location,
+          path: join(dep.root.localPath, nmFolder, dep.name),
+          realpath: target.path,
+          name: toKey,
+          resolved: dep.resolved,
+          top: { path: dep.root.localPath },
+          children: [],
+          fsChildren: [],
+          isLink: true,
+          isStoreLink: true,
+          isRoot: false,
+          package: { _id: 'abc', bundleDependencies: undefined, deprecated: undefined, bin: target.package.bin, scripts: dep.package.scripts },
+          target,
+        }
+        const newEdge1 = { optional, from, to: link }
+        from.edgesOut.set(dep.name, newEdge1)
+        link.edgesIn.add(newEdge1)
+        const newEdge2 = { optional: false, from: link, to: target }
+        link.edgesOut.set(dep.name, newEdge2)
+        target.edgesIn.add(newEdge2)
+        root.children.push(link)
+      }
+
+      for (const dep of node.localDependencies) {
+        processEdges(dep, false)
+        // nonOptional, local
+        processDeps(dep, false, false)
+      }
+      for (const dep of node.externalDependencies) {
+        processEdges(dep, true)
+        // nonOptional, external
+        processDeps(dep, false, true)
+      }
+      for (const dep of node.externalOptionalDependencies) {
+        processEdges(dep, true)
+        // optional, external
+        processDeps(dep, true, true)
+      }
+    }
+
+    processEdges(proxiedIdealTree, false)
+    for (const node of proxiedIdealTree.workspaces) {
+      processEdges(node, false)
+    }
+    root.children.forEach(c => c.parent = root)
+    root.children.forEach(c => c.root = root)
+    root.root = root
+    root.target = root
+    return root
+  }
+}

package/lib/arborist/load-virtual.js

@@ -1,5 +1,6 @@
 // mixin providing the loadVirtual method
 const localeCompare = require('@isaacs/string-locale-compare')('en')
+const mapWorkspaces = require('@npmcli/map-workspaces')
 
 const { resolve } = require('path')
 
@@ -21,7 +22,6 @@ const loadRoot = Symbol('loadRoot')
 const loadNode = Symbol('loadVirtualNode')
 const loadLink = Symbol('loadVirtualLink')
 const loadWorkspaces = Symbol.for('loadWorkspaces')
-const loadWorkspacesVirtual = Symbol.for('loadWorkspacesVirtual')
 const flagsSuspect = Symbol.for('flagsSuspect')
 const reCalcDepFlags = Symbol('reCalcDepFlags')
 const checkRootEdges = Symbol('checkRootEdges')
@@ -157,7 +157,7 @@ module.exports = cls => class VirtualLoader extends cls {
     }
 
     const lockWS = []
-    const workspaces = this[loadWorkspacesVirtual]({
+    const workspaces = mapWorkspaces.virtual({
       cwd: this.path,
       lockfile: s.data,
     })

package/lib/arborist/load-workspaces.js

@@ -1,33 +1,19 @@
 const mapWorkspaces = require('@npmcli/map-workspaces')
 
-const _appendWorkspaces = Symbol('appendWorkspaces')
 // shared ref used by other mixins/Arborist
 const _loadWorkspaces = Symbol.for('loadWorkspaces')
-const _loadWorkspacesVirtual = Symbol.for('loadWorkspacesVirtual')
 
 module.exports = cls => class MapWorkspaces extends cls {
-  [_appendWorkspaces] (node, workspaces) {
-    if (node && workspaces.size) {
-      node.workspaces = workspaces
-    }
-
-    return node
-  }
-
   async [_loadWorkspaces] (node) {
-    if (node.workspaces) {
-      return node
-    }
-
     const workspaces = await mapWorkspaces({
       cwd: node.path,
       pkg: node.package,
     })
 
-    return this[_appendWorkspaces](node, workspaces)
-  }
+    if (node && workspaces.size) {
+      node.workspaces = workspaces
+    }
 
-  [_loadWorkspacesVirtual] (opts) {
-    return mapWorkspaces.virtual(opts)
+    return node
   }
 }

package/lib/arborist/rebuild.js

@@ -89,6 +89,7 @@ module.exports = cls => class Builder extends cls {
     const {
       depNodes,
       linkNodes,
+      storeNodes,
     } = this[_retrieveNodesByType](nodes)
 
     // build regular deps
@@ -99,6 +100,10 @@ module.exports = cls => class Builder extends cls {
       this[_resetQueues]()
       await this[_build](linkNodes, { type: 'links' })
     }
+    if (storeNodes.size) {
+      this[_resetQueues]()
+      await this[_build](storeNodes, { type: 'storelinks' })
+    }
 
     process.emit('timeEnd', 'build')
   }
@@ -130,9 +135,12 @@ module.exports = cls => class Builder extends cls {
   [_retrieveNodesByType] (nodes) {
     const depNodes = new Set()
     const linkNodes = new Set()
+    const storeNodes = new Set()
 
     for (const node of nodes) {
-      if (node.isLink) {
+      if (node.isStoreLink) {
+        storeNodes.add(node)
+      } else if (node.isLink) {
         linkNodes.add(node)
       } else {
         depNodes.add(node)
@@ -154,6 +162,7 @@ module.exports = cls => class Builder extends cls {
     return {
       depNodes,
       linkNodes,
+      storeNodes,
     }
   }
 

package/lib/arborist/reify.js

@@ -1,5 +1,4 @@
 // mixin implementing the reify method
-
 const onExit = require('../signal-handling.js')
 const pacote = require('pacote')
 const AuditReport = require('../audit-report.js')
@@ -10,8 +9,9 @@ const debug = require('../debug.js')
 const walkUp = require('walk-up-path')
 const log = require('proc-log')
 const hgi = require('hosted-git-info')
+const rpj = require('read-package-json-fast')
 
-const { dirname, resolve, relative } = require('path')
+const { dirname, resolve, relative, join } = require('path')
 const { depth: dfwalk } = require('treeverse')
 const {
   lstat,
@@ -106,6 +106,8 @@ const _resolvedAdd = Symbol.for('resolvedAdd')
 const _usePackageLock = Symbol.for('usePackageLock')
 const _formatPackageLock = Symbol.for('formatPackageLock')
 
+const _createIsolatedTree = Symbol.for('createIsolatedTree')
+
 module.exports = cls => class Reifier extends cls {
   constructor (options) {
     super(options)
@@ -138,6 +140,8 @@ module.exports = cls => class Reifier extends cls {
 
   // public method
   async reify (options = {}) {
+    const linked = (options.installStrategy || this.options.installStrategy) === 'linked'
+
     if (this[_packageLockOnly] && this[_global]) {
       const er = new Error('cannot generate lockfile for global packages')
       er.code = 'ESHRINKWRAPGLOBAL'
@@ -154,8 +158,22 @@ module.exports = cls => class Reifier extends cls {
     process.emit('time', 'reify')
     await this[_validatePath]()
     await this[_loadTrees](options)
+
+    const oldTree = this.idealTree
+    if (linked) {
+      // swap out the tree with the isolated tree
+      // this is currently technical debt which will be resolved in a refactor
+      // of Node/Link trees
+      log.warn('reify', 'The "linked" install strategy is EXPERIMENTAL and may contain bugs.')
+      this.idealTree = await this[_createIsolatedTree](this.idealTree)
+    }
     await this[_diffTrees]()
     await this[_reifyPackages]()
+    if (linked) {
+      // swap back in the idealTree
+      // so that the lockfile is preserved
+      this.idealTree = oldTree
+    }
     await this[_saveIdealTree](options)
     await this[_copyIdealToActual]()
     // This is a very bad pattern and I can't wait to stop doing it
@@ -634,44 +652,40 @@ module.exports = cls => class Reifier extends cls {
   }
 
   async [_extractOrLink] (node) {
-    // in normal cases, node.resolved should *always* be set by now.
-    // however, it is possible when a lockfile is damaged, or very old,
-    // or in some other race condition bugs in npm v6, that a previously
-    // bundled dependency will have just a version, but no resolved value,
-    // and no 'bundled: true' setting.
-    // Do the best with what we have, or else remove it from the tree
-    // entirely, since we can't possibly reify it.
-    let res = null
-    if (node.resolved) {
-      const registryResolved = this[_registryResolved](node.resolved)
-      if (registryResolved) {
-        res = `${node.name}@${registryResolved}`
-      }
-    } else if (node.packageName && node.version) {
-      res = `${node.packageName}@${node.version}`
-    }
-
-    // no idea what this thing is.  remove it from the tree.
-    if (!res) {
-      const warning = 'invalid or damaged lockfile detected\n' +
-        'please re-try this operation once it completes\n' +
-        'so that the damage can be corrected, or perform\n' +
-        'a fresh install with no lockfile if the problem persists.'
-      log.warn('reify', warning)
-      log.verbose('reify', 'unrecognized node in tree', node.path)
-      node.parent = null
-      node.fsParent = null
-      this[_addNodeToTrashList](node)
-      return
-    }
-
     const nm = resolve(node.parent.path, 'node_modules')
     await this[_validateNodeModules](nm)
 
-    if (node.isLink) {
-      await rm(node.path, { recursive: true, force: true })
-      await this[_symlink](node)
-    } else {
+    if (!node.isLink) {
+      // in normal cases, node.resolved should *always* be set by now.
+      // however, it is possible when a lockfile is damaged, or very old,
+      // or in some other race condition bugs in npm v6, that a previously
+      // bundled dependency will have just a version, but no resolved value,
+      // and no 'bundled: true' setting.
+      // Do the best with what we have, or else remove it from the tree
+      // entirely, since we can't possibly reify it.
+      let res = null
+      if (node.resolved) {
+        const registryResolved = this[_registryResolved](node.resolved)
+        if (registryResolved) {
+          res = `${node.name}@${registryResolved}`
+        }
+      } else if (node.package.name && node.version) {
+        res = `${node.package.name}@${node.version}`
+      }
+
+      // no idea what this thing is.  remove it from the tree.
+      if (!res) {
+        const warning = 'invalid or damaged lockfile detected\n' +
+          'please re-try this operation once it completes\n' +
+          'so that the damage can be corrected, or perform\n' +
+          'a fresh install with no lockfile if the problem persists.'
+        log.warn('reify', warning)
+        log.verbose('reify', 'unrecognized node in tree', node.path)
+        node.parent = null
+        node.fsParent = null
+        this[_addNodeToTrashList](node)
+        return
+      }
       await debug(async () => {
         const st = await lstat(node.path).catch(e => null)
         if (st && !st.isDirectory()) {
@@ -688,7 +702,17 @@ module.exports = cls => class Reifier extends cls {
         resolved: node.resolved,
         integrity: node.integrity,
       })
+      // store nodes don't use Node class so node.package doesn't get updated
+      if (node.isInStore) {
+        const pkg = await rpj(join(node.path, 'package.json'))
+        node.package.scripts = pkg.scripts
+      }
+      return
     }
+
+    // node.isLink
+    await rm(node.path, { recursive: true, force: true })
+    await this[_symlink](node)
   }
 
   async [_symlink] (node) {

package/lib/link.js

@@ -8,7 +8,7 @@ const _delistFromMeta = Symbol.for('_delistFromMeta')
 const _refreshLocation = Symbol.for('_refreshLocation')
 class Link extends Node {
   constructor (options) {
-    const { root, realpath, target, parent, fsParent } = options
+    const { root, realpath, target, parent, fsParent, isStoreLink } = options
 
     if (!realpath && !(target && target.path)) {
       throw new TypeError('must provide realpath for Link node')
@@ -23,6 +23,8 @@ class Link extends Node {
       : null),
     })
 
+    this.isStoreLink = isStoreLink || false
+
     if (target) {
       this.target = target
     } else if (this.realpath === this.root.path) {

package/lib/node.js

@@ -91,6 +91,7 @@ class Node {
       installLinks = false,
       legacyPeerDeps = false,
       linksIn,
+      isInStore = false,
       hasShrinkwrap,
       overrides,
       loadOverrides = false,
@@ -113,6 +114,7 @@ class Node {
     this[_workspaces] = null
 
     this.errors = error ? [error] : []
+    this.isInStore = isInStore
 
     // this will usually be null, except when modeling a
     // package's dependencies in a virtual root.

package/lib/override-resolves.js

@@ -1,4 +1,4 @@
-function overrideResolves (resolved, opts = {}) {
+function overrideResolves (resolved, opts) {
   const { omitLockfileRegistryResolved = false } = opts
 
   if (omitLockfileRegistryResolved) {

package/lib/override-set.js

@@ -50,9 +50,36 @@ class OverrideSet {
         continue
       }
 
-      if (semver.intersects(edge.spec, rule.keySpec)) {
+      // if keySpec is * we found our override
+      if (rule.keySpec === '*') {
         return rule
       }
+
+      let spec = npa(`${edge.name}@${edge.spec}`)
+      if (spec.type === 'alias') {
+        spec = spec.subSpec
+      }
+
+      if (spec.type === 'git') {
+        if (spec.gitRange && semver.intersects(spec.gitRange, rule.keySpec)) {
+          return rule
+        }
+
+        continue
+      }
+
+      if (spec.type === 'range' || spec.type === 'version') {
+        if (semver.intersects(spec.fetchSpec, rule.keySpec)) {
+          return rule
+        }
+
+        continue
+      }
+
+      // if we got this far, the spec type is one of tag, directory or file
+      // which means we have no real way to make version comparisons, so we
+      // just accept the override
+      return rule
     }
 
     return this

package/lib/place-dep.js

@@ -9,6 +9,7 @@
 
 const localeCompare = require('@isaacs/string-locale-compare')('en')
 const log = require('proc-log')
+const { cleanUrl } = require('npm-registry-fetch')
 const deepestNestingTarget = require('./deepest-nesting-target.js')
 const CanPlaceDep = require('./can-place-dep.js')
 const {
@@ -187,7 +188,7 @@ class PlaceDep {
       `${this.dep.name}@${this.dep.version}`,
       this.canPlace.description,
       `for: ${this.edge.from.package._id || this.edge.from.location}`,
-      `want: ${this.edge.spec || '*'}`
+      `want: ${cleanUrl(this.edge.spec || '*')}`
     )
 
     const placementType = this.canPlace.canPlace === CONFLICT

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@npmcli/arborist",
-  "version": "6.1.5",
+  "version": "6.2.0",
   "description": "Manage node_modules trees",
   "dependencies": {
     "@isaacs/string-locale-compare": "^1.1.0",
@@ -39,16 +39,18 @@
   },
   "devDependencies": {
     "@npmcli/eslint-config": "^4.0.0",
-    "@npmcli/template-oss": "4.11.0",
+    "@npmcli/template-oss": "4.11.1",
     "benchmark": "^2.1.4",
     "chalk": "^4.1.0",
-    "minify-registry-metadata": "^2.1.0",
+    "minify-registry-metadata": "^3.0.0",
     "nock": "^13.2.0",
-    "tap": "^16.0.1",
+    "tap": "^16.3.2",
+    "tar-stream": "^3.0.0",
     "tcompare": "^5.0.6"
   },
   "scripts": {
     "test": "tap",
+    "test-only": "tap --only",
     "posttest": "node ../.. run lint",
     "snap": "tap",
     "postsnap": "npm run lintfix",
@@ -81,7 +83,6 @@
   "tap": {
     "color": true,
     "after": "test/fixtures/cleanup.js",
-    "coverage-map": "map.js",
     "test-env": [
       "NODE_OPTIONS=--no-warnings",
       "LC_ALL=sk"
@@ -101,7 +102,7 @@
   },
   "templateOSS": {
     "//@npmcli/template-oss": "This file is partially managed by @npmcli/template-oss. Edits may be overwritten.",
-    "version": "4.11.0",
+    "version": "4.11.1",
     "content": "../../scripts/template-oss/index.js"
   }
 }