@pnpm/lockfile.filtering 1100.1.14 → 1100.2.0

package/CHANGELOG.md

@@ -1,5 +1,21 @@
 # @pnpm/filter-lockfile
 
+## 1100.2.0
+
+### Minor Changes
+
+- Fixed an installed optional dependency being left without one of its own required dependencies. When a package reached through `optionalDependencies` is installable on the current system but one of its regular `dependencies` is not, a lockfile-based install skipped that dependency and installed the parent anyway, so importing the parent failed with `MODULE_NOT_FOUND`. The dependency is now installed, and an install-check warning reports the incompatibility. A dependency is still only skipped when every path to it is optional, or when the package that pulls it in was itself skipped [#13286](https://github.com/pnpm/pnpm/issues/13286).
+
+### Patch Changes
+
+- Updated dependencies:
+  - @pnpm/config.package-is-installable@1100.1.0
+  - @pnpm/deps.path@1100.0.12
+  - @pnpm/lockfile.types@1100.0.17
+  - @pnpm/lockfile.utils@1100.1.6
+  - @pnpm/lockfile.walker@1100.0.17
+  - @pnpm/types@1101.7.0
+
 ## 1100.1.14
 
 ### Patch Changes

package/lib/filterLockfileByImportersAndEngine.d.ts

@@ -1,8 +1,19 @@
 import type { LockfileObject } from '@pnpm/lockfile.types';
-import type { DependenciesField, ProjectId, SupportedArchitectures } from '@pnpm/types';
+import type { DependenciesField, DepPath, ProjectId, SupportedArchitectures } from '@pnpm/types';
 export interface FilterLockfileResult {
     lockfile: LockfileObject;
     selectedImporterIds: ProjectId[];
+    /**
+     * Dep paths reached by a non-optional edge from an importer or from a
+     * package that is itself part of the install. Their installability is
+     * evaluated as non-optional — an incompatible one fails the install under
+     * `engineStrict` instead of being skipped — even when the lockfile marks
+     * the snapshot `optional: true` because the subtree happens to hang off an
+     * optional dependency. Downstream consumers must classify by this set
+     * rather than by `pkgSnapshot.optional` so every stage of a headless
+     * install agrees with the resolver, which classifies by edge too.
+     */
+    requiredDepPaths: Set<DepPath>;
 }
 export declare function filterLockfileByEngine(lockfile: LockfileObject, opts: FilterLockfileOptions): FilterLockfileResult;
 export interface FilterLockfileOptions {

package/lib/filterLockfileByImportersAndEngine.js

@@ -1,4 +1,4 @@
-import { packageIsInstallable } from '@pnpm/config.package-is-installable';
+import { checkPackageInstallability, packageIsInstallable, } from '@pnpm/config.package-is-installable';
 import { WANTED_LOCKFILE } from '@pnpm/constants';
 import * as dp from '@pnpm/deps.path';
 import { LockfileMissingDependencyError } from '@pnpm/error';
@@ -13,13 +13,13 @@ export function filterLockfileByEngine(lockfile, opts) {
 }
 export function filterLockfileByImportersAndEngine(lockfile, importerIds, opts) {
     const importerIdSet = new Set(importerIds);
-    const directDepPaths = toImporterDepPaths(lockfile, importerIds, {
+    const directDepEdges = toImporterDepPaths(lockfile, importerIds, {
         include: opts.include,
         importerIdSet,
         skipRuntimes: opts.skipRuntimes,
     });
-    const packages = lockfile.packages != null
-        ? pickPkgsWithAllDeps(lockfile, directDepPaths, importerIdSet, {
+    const { packages, requiredDepPaths } = lockfile.packages != null
+        ? pickPkgsWithAllDeps(lockfile, directDepEdges, importerIdSet, {
             currentEngine: opts.currentEngine,
             engineStrict: opts.engineStrict,
             failOnMissingDependencies: opts.failOnMissingDependencies,
@@ -30,7 +30,7 @@ export function filterLockfileByImportersAndEngine(lockfile, importerIds, opts)
             skipRuntimes: opts.skipRuntimes,
             supportedArchitectures: opts.supportedArchitectures,
         })
-        : {};
+        : { packages: {}, requiredDepPaths: new Set() };
     const importers = mapValues((importer) => {
         const newImporter = filterImporter(importer, opts.include, { skipRuntimes: opts.skipRuntimes });
         if (newImporter.optionalDependencies != null) {
@@ -48,18 +48,133 @@ export function filterLockfileByImportersAndEngine(lockfile, importerIds, opts)
             packages,
         },
         selectedImporterIds: Array.from(importerIdSet),
+        requiredDepPaths,
     };
 }
-function pickPkgsWithAllDeps(lockfile, depPaths, importerIdSet, opts) {
-    const pickedPackages = {};
-    pkgAllDeps({ lockfile, pickedPackages, importerIdSet }, depPaths, true, opts);
-    return pickedPackages;
+function pickPkgsWithAllDeps(lockfile, depEdges, importerIdSet, opts) {
+    const ctx = {
+        lockfile,
+        pickedPackages: {},
+        importerIdSet,
+        installed: new Set(),
+        requiredDepPaths: new Set(),
+        evaluated: [],
+        edgesByDepPath: new Map(),
+    };
+    classifyDeps(ctx, depEdges, opts);
+    reportInstallability(ctx, opts);
+    pickSkippedDeps(ctx, depEdges, opts);
+    return { packages: ctx.pickedPackages, requiredDepPaths: ctx.requiredDepPaths };
 }
-function pkgAllDeps(ctx, depPaths, parentIsInstallable, opts) {
-    for (const depPath of depPaths) {
-        if (ctx.pickedPackages[depPath])
+/**
+ * Work out which packages the install actually reaches, and over which kind
+ * of edge.
+ *
+ * An edge is only expanded once its source is known to be part of the
+ * install, so the optionality of the edge — not the `optional` flag the
+ * resolver propagated onto the snapshot — decides whether an incompatible
+ * package is skipped: only a package whose every inbound edge is optional
+ * stays out of the install.
+ *
+ * Traversal is breadth-first over edges rather than depth-first over
+ * packages so the verdict does not depend on which edge happens to reach a
+ * package first: a package left out on an optional edge is reconsidered when
+ * a non-optional one arrives later. Nothing is reported from here — a
+ * package can be visited once per inbound edge, and its installability must
+ * be reported exactly once, which [reportInstallability] does afterwards.
+ */
+function classifyDeps(ctx, depEdges, opts) {
+    const incompatible = new Map();
+    // Walked with a cursor rather than `shift()`, which is O(n) per dequeue on
+    // a JS array and would make the walk quadratic on a large lockfile.
+    const queue = [...depEdges];
+    for (let next = 0; next < queue.length; next++) {
+        const { depPath, optional } = queue[next];
+        if (!optional)
+            ctx.requiredDepPaths.add(depPath);
+        if (ctx.installed.has(depPath))
             continue;
         const pkgSnapshot = ctx.lockfile.packages[depPath];
+        // Missing entries are reported by the closure pass, which reaches every
+        // dep path this one does.
+        if (!pkgSnapshot)
+            continue;
+        if (!incompatible.has(depPath)) {
+            ctx.evaluated.push(depPath);
+            // TODO: depPath is not the package ID. Should be fixed
+            incompatible.set(depPath, !opts.includeIncompatiblePackages && checkPackageInstallability(pkgSnapshot.id ?? depPath, toInstallabilityManifest(depPath, pkgSnapshot), {
+                nodeVersion: opts.currentEngine.nodeVersion,
+                optional: true,
+                supportedArchitectures: opts.supportedArchitectures,
+            }) != null);
+        }
+        if (optional && incompatible.get(depPath))
+            continue;
+        ctx.installed.add(depPath);
+        ctx.pickedPackages[depPath] = pkgSnapshot;
+        const edges = nextDepEdges(ctx, pkgSnapshot, opts);
+        ctx.edgesByDepPath.set(depPath, edges);
+        // Appended one by one: `push(...edges)` passes each edge as its own
+        // argument and overflows the engine's argument limit on a wide enough
+        // dependency list.
+        for (const edge of edges) {
+            queue.push(edge);
+        }
+    }
+}
+/**
+ * Report each reached package once, against the edge kind it was classified
+ * by. This is where an incompatible package that a non-optional edge reaches
+ * fails the install under `engineStrict`, and where a skipped optional gets
+ * its `skippedOptionalDependencyLogger` entry.
+ */
+function reportInstallability(ctx, opts) {
+    for (const depPath of ctx.evaluated) {
+        const pkgSnapshot = ctx.lockfile.packages[depPath];
+        const installable = opts.includeIncompatiblePackages ||
+            packageIsInstallable(pkgSnapshot.id ?? depPath, toInstallabilityManifest(depPath, pkgSnapshot), {
+                // A subtree hanging off an `optionalDependencies` entry stays
+                // best-effort: the dependency is installed so its dependent is not
+                // left broken, but an incompatibility inside it does not fail the
+                // install. Only a package no optional path reaches is fatal here.
+                engineStrict: opts.engineStrict && pkgSnapshot.optional !== true,
+                lockfileDir: opts.lockfileDir,
+                nodeVersion: opts.currentEngine.nodeVersion,
+                optional: !ctx.installed.has(depPath) || !ctx.requiredDepPaths.has(depPath),
+                supportedArchitectures: opts.supportedArchitectures,
+            }) !== false;
+        if (installable) {
+            opts.skipped.delete(depPath);
+        }
+        else {
+            opts.skipped.add(depPath);
+        }
+    }
+}
+function toInstallabilityManifest(depPath, pkgSnapshot) {
+    return {
+        ...nameVerFromPkgSnapshot(depPath, pkgSnapshot),
+        cpu: pkgSnapshot.cpu,
+        engines: pkgSnapshot.engines,
+        os: pkgSnapshot.os,
+        libc: pkgSnapshot.libc,
+    };
+}
+/**
+ * Extend the picked set over the packages that are reachable but not part of
+ * the install — an incompatible optional and everything below it. They stay
+ * in the filtered lockfile and are recorded as skipped, so a later install on
+ * a host that does support them can pick them up without a re-resolution.
+ */
+function pickSkippedDeps(ctx, depEdges, opts) {
+    const queue = depEdges.map(({ depPath }) => depPath);
+    const visited = new Set();
+    for (let next = 0; next < queue.length; next++) {
+        const depPath = queue[next];
+        if (visited.has(depPath))
+            continue;
+        visited.add(depPath);
+        const pkgSnapshot = ctx.lockfile.packages[depPath];
         if (!pkgSnapshot && !depPath.startsWith('link:')) {
             if (opts.failOnMissingDependencies) {
                 throw new LockfileMissingDependencyError(depPath);
@@ -67,87 +182,73 @@ function pkgAllDeps(ctx, depPaths, parentIsInstallable, opts) {
             lockfileLogger.debug(`No entry for "${depPath}" in ${WANTED_LOCKFILE}`);
             continue;
         }
-        let installable;
-        if (!parentIsInstallable) {
-            installable = false;
+        if (!ctx.installed.has(depPath)) {
             if (!ctx.pickedPackages[depPath] && pkgSnapshot.optional === true) {
                 opts.skipped.add(depPath);
             }
+            ctx.pickedPackages[depPath] = pkgSnapshot;
         }
-        else {
-            const pkg = {
-                ...nameVerFromPkgSnapshot(depPath, pkgSnapshot),
-                cpu: pkgSnapshot.cpu,
-                engines: pkgSnapshot.engines,
-                os: pkgSnapshot.os,
-                libc: pkgSnapshot.libc,
-            };
-            // TODO: depPath is not the package ID. Should be fixed
-            installable =
-                opts.includeIncompatiblePackages ||
-                    packageIsInstallable(pkgSnapshot.id ?? depPath, pkg, {
-                        engineStrict: opts.engineStrict,
-                        lockfileDir: opts.lockfileDir,
-                        nodeVersion: opts.currentEngine.nodeVersion,
-                        optional: pkgSnapshot.optional === true,
-                        supportedArchitectures: opts.supportedArchitectures,
-                    }) !== false;
-            if (!installable) {
-                if (!ctx.pickedPackages[depPath] && pkgSnapshot.optional === true) {
-                    opts.skipped.add(depPath);
-                }
-            }
-            else {
-                opts.skipped.delete(depPath);
-            }
+        // `visited` guarantees one pass per dep path, so a cached entry is
+        // released as soon as it is consumed rather than being retained until the
+        // whole walk ends.
+        const edges = ctx.edgesByDepPath.get(depPath) ?? nextDepEdges(ctx, pkgSnapshot, opts);
+        ctx.edgesByDepPath.delete(depPath);
+        for (const edge of edges) {
+            queue.push(edge.depPath);
         }
-        ctx.pickedPackages[depPath] = pkgSnapshot;
-        const { depPaths: nextRelDepPaths, importerIds: additionalImporterIds } = parseDepRefs(Object.entries({
-            ...pkgSnapshot.dependencies,
-            ...(opts.include.optionalDependencies
-                ? pkgSnapshot.optionalDependencies
-                : {}),
-        }), ctx.lockfile);
-        additionalImporterIds.forEach((importerId) => ctx.importerIdSet.add(importerId));
-        nextRelDepPaths.push(...toImporterDepPaths(ctx.lockfile, additionalImporterIds, {
+    }
+}
+/** The outbound edges of a package, tagged with the optionality of each. */
+function nextDepEdges(ctx, pkgSnapshot, opts) {
+    const { depEdges, importerIds } = parseDepRefs([
+        ...toDepRefs(pkgSnapshot.dependencies, false),
+        ...(opts.include.optionalDependencies ? toDepRefs(pkgSnapshot.optionalDependencies, true) : []),
+    ], ctx.lockfile);
+    for (const importerId of importerIds) {
+        ctx.importerIdSet.add(importerId);
+    }
+    return [
+        ...depEdges,
+        ...toImporterDepPaths(ctx.lockfile, importerIds, {
             include: opts.include,
             importerIdSet: ctx.importerIdSet,
             skipRuntimes: opts.skipRuntimes,
-        }));
-        pkgAllDeps(ctx, nextRelDepPaths, installable, opts);
-    }
+        }),
+    ];
 }
 function toImporterDepPaths(lockfile, importerIds, opts) {
     const importerDeps = importerIds
         .map(importerId => lockfile.importers[importerId])
-        .map(importer => ({
-        ...(opts.include.dependencies ? importer.dependencies : {}),
-        ...(opts.include.devDependencies ? importer.devDependencies : {}),
-        ...(opts.include.optionalDependencies
-            ? importer.optionalDependencies
-            : {}),
-    }))
-        .map(Object.entries)
-        .map(entries => opts.skipRuntimes ? entries.filter(([, ref]) => !ref.startsWith('runtime:')) : entries);
-    let { depPaths, importerIds: nextImporterIds } = parseDepRefs(unnest(importerDeps), lockfile);
+        .map(importer => [
+        ...(opts.include.dependencies ? toDepRefs(importer.dependencies, false) : []),
+        ...(opts.include.devDependencies ? toDepRefs(importer.devDependencies, false) : []),
+        ...(opts.include.optionalDependencies ? toDepRefs(importer.optionalDependencies, true) : []),
+    ])
+        .map(refs => opts.skipRuntimes ? refs.filter(({ ref }) => !ref.startsWith('runtime:')) : refs);
+    let { depEdges, importerIds: nextImporterIds } = parseDepRefs(unnest(importerDeps), lockfile);
     if (!nextImporterIds.length) {
-        return depPaths;
+        return depEdges;
     }
     nextImporterIds = nextImporterIds.filter(importerId => !opts.importerIdSet.has(importerId));
     for (const importerId of nextImporterIds) {
         opts.importerIdSet.add(importerId);
     }
     return [
-        ...depPaths,
+        ...depEdges,
         ...toImporterDepPaths(lockfile, nextImporterIds, opts),
     ];
 }
-function parseDepRefs(refsByPkgNames, lockfile) {
+function toDepRefs(refsByPkgNames, optional) {
+    if (refsByPkgNames == null)
+        return [];
+    return Object.entries(refsByPkgNames).map(([pkgName, ref]) => ({ pkgName, ref, optional }));
+}
+function parseDepRefs(depRefs, lockfile) {
     const acc = {
-        depPaths: [],
+        depEdges: [],
         importerIds: [],
     };
-    for (const [pkgName, ref] of refsByPkgNames) {
+    for (const { pkgName, ref, optional } of depRefs) {
         if (ref.startsWith('link:')) {
             const importerId = ref.substring(5);
             if (lockfile.importers[importerId]) {
@@ -158,7 +259,7 @@ function parseDepRefs(refsByPkgNames, lockfile) {
         const depPath = dp.refToRelative(ref, pkgName);
         if (depPath == null)
             continue;
-        acc.depPaths.push(depPath);
+        acc.depEdges.push({ depPath, optional });
     }
     return acc;
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pnpm/lockfile.filtering",
-  "version": "1100.1.14",
+  "version": "1100.2.0",
   "description": "Filters a lockfile",
   "keywords": [
     "pnpm",
@@ -29,14 +29,14 @@
     "!*.map"
   ],
   "dependencies": {
-    "@pnpm/config.package-is-installable": "1100.0.16",
+    "@pnpm/config.package-is-installable": "1100.1.0",
     "@pnpm/constants": "1100.0.1",
-    "@pnpm/deps.path": "1100.0.11",
+    "@pnpm/deps.path": "1100.0.12",
     "@pnpm/error": "1100.1.0",
-    "@pnpm/lockfile.types": "1100.0.16",
-    "@pnpm/lockfile.utils": "1100.1.5",
-    "@pnpm/lockfile.walker": "1100.0.16",
-    "@pnpm/types": "1101.6.0",
+    "@pnpm/lockfile.types": "1100.0.17",
+    "@pnpm/lockfile.utils": "1100.1.6",
+    "@pnpm/lockfile.walker": "1100.0.17",
+    "@pnpm/types": "1101.7.0",
     "ramda": "npm:@pnpm/ramda@0.28.1"
   },
   "peerDependencies": {
@@ -44,9 +44,9 @@
   },
   "devDependencies": {
     "@jest/globals": "30.4.1",
-    "@pnpm/lockfile.filtering": "1100.1.14",
+    "@pnpm/lockfile.filtering": "1100.2.0",
     "@pnpm/logger": "1100.0.0",
-    "@types/ramda": "0.31.1",
+    "@types/ramda": "0.32.0",
     "detect-libc": "^2.1.2",
     "tempy": "3.0.0",
     "write-yaml-file": "^6.0.0",