@pnpm/lockfile.filtering 1100.1.13 → 1100.2.0

package/CHANGELOG.md

@@ -1,5 +1,37 @@
 # @pnpm/filter-lockfile
 
+## 1100.2.0
+
+### Minor Changes
+
+- Fixed an installed optional dependency being left without one of its own required dependencies. When a package reached through `optionalDependencies` is installable on the current system but one of its regular `dependencies` is not, a lockfile-based install skipped that dependency and installed the parent anyway, so importing the parent failed with `MODULE_NOT_FOUND`. The dependency is now installed, and an install-check warning reports the incompatibility. A dependency is still only skipped when every path to it is optional, or when the package that pulls it in was itself skipped [#13286](https://github.com/pnpm/pnpm/issues/13286).
+
+### Patch Changes
+
+- Updated dependencies:
+  - @pnpm/config.package-is-installable@1100.1.0
+  - @pnpm/deps.path@1100.0.12
+  - @pnpm/lockfile.types@1100.0.17
+  - @pnpm/lockfile.utils@1100.1.6
+  - @pnpm/lockfile.walker@1100.0.17
+  - @pnpm/types@1101.7.0
+
+## 1100.1.14
+
+### Patch Changes
+
+- Republished every package: the tarballs published by the v11.13.1 through v11.16.0 releases were missing most of their compiled files due to a packing bug [#13164](https://github.com/pnpm/pnpm/issues/13164).
+
+- Updated dependencies:
+  - @pnpm/config.package-is-installable@1100.0.16
+  - @pnpm/constants@1100.0.1
+  - @pnpm/deps.path@1100.0.11
+  - @pnpm/error@1100.1.0
+  - @pnpm/lockfile.types@1100.0.16
+  - @pnpm/lockfile.utils@1100.1.5
+  - @pnpm/lockfile.walker@1100.0.16
+  - @pnpm/types@1101.6.0
+
 ## 1100.1.13
 
 ### Patch Changes

package/lib/filterImporter.d.ts

@@ -0,0 +1,7 @@
+import type { ProjectSnapshot } from '@pnpm/lockfile.types';
+import type { DependenciesField } from '@pnpm/types';
+export declare function filterImporter(importer: ProjectSnapshot, include: {
+    [dependenciesField in DependenciesField]: boolean;
+}, opts?: {
+    skipRuntimes?: boolean;
+}): ProjectSnapshot;

package/lib/filterImporter.js

@@ -0,0 +1,23 @@
+export function filterImporter(importer, include, opts) {
+    const skipRuntimes = opts?.skipRuntimes === true;
+    return {
+        dependencies: !include.dependencies ? {} : pickNonRuntime(importer.dependencies, skipRuntimes),
+        devDependencies: !include.devDependencies ? {} : pickNonRuntime(importer.devDependencies, skipRuntimes),
+        optionalDependencies: !include.optionalDependencies ? {} : pickNonRuntime(importer.optionalDependencies, skipRuntimes),
+        specifiers: pickNonRuntime(importer.specifiers, skipRuntimes),
+    };
+}
+function pickNonRuntime(deps, skipRuntimes) {
+    if (!deps)
+        return {};
+    if (!skipRuntimes)
+        return deps;
+    const result = {};
+    for (const [name, ref] of Object.entries(deps)) {
+        if (!ref.startsWith('runtime:')) {
+            result[name] = ref;
+        }
+    }
+    return result;
+}
+//# sourceMappingURL=filterImporter.js.map

package/lib/filterLockfile.d.ts

@@ -0,0 +1,9 @@
+import type { LockfileObject } from '@pnpm/lockfile.types';
+import type { DependenciesField, DepPath } from '@pnpm/types';
+export declare function filterLockfile(lockfile: LockfileObject, opts: {
+    include: {
+        [dependenciesField in DependenciesField]: boolean;
+    };
+    skipped: Set<DepPath>;
+    skipRuntimes?: boolean;
+}): LockfileObject;

package/lib/filterLockfile.js

@@ -0,0 +1,8 @@
+import { filterLockfileByImporters } from './filterLockfileByImporters.js';
+export function filterLockfile(lockfile, opts) {
+    return filterLockfileByImporters(lockfile, Object.keys(lockfile.importers), {
+        ...opts,
+        failOnMissingDependencies: false,
+    });
+}
+//# sourceMappingURL=filterLockfile.js.map

package/lib/filterLockfileByImporters.d.ts

@@ -0,0 +1,10 @@
+import type { LockfileObject } from '@pnpm/lockfile.types';
+import type { DependenciesField, DepPath, ProjectId } from '@pnpm/types';
+export declare function filterLockfileByImporters(lockfile: LockfileObject, importerIds: ProjectId[], opts: {
+    include: {
+        [dependenciesField in DependenciesField]: boolean;
+    };
+    skipped: Set<DepPath>;
+    skipRuntimes?: boolean;
+    failOnMissingDependencies: boolean;
+}): LockfileObject;

package/lib/filterLockfileByImporters.js

@@ -0,0 +1,36 @@
+import { WANTED_LOCKFILE } from '@pnpm/constants';
+import { LockfileMissingDependencyError } from '@pnpm/error';
+import { lockfileWalker } from '@pnpm/lockfile.walker';
+import { logger } from '@pnpm/logger';
+import { filterImporter } from './filterImporter.js';
+const lockfileLogger = logger('lockfile');
+export function filterLockfileByImporters(lockfile, importerIds, opts) {
+    const importers = { ...lockfile.importers };
+    for (const importerId of importerIds) {
+        importers[importerId] = filterImporter(lockfile.importers[importerId], opts.include, { skipRuntimes: opts.skipRuntimes });
+    }
+    const packages = {};
+    if (lockfile.packages != null) {
+        pkgAllDeps(lockfileWalker({ ...lockfile, importers }, importerIds, { include: opts.include, skipped: opts.skipped }).step, packages, {
+            failOnMissingDependencies: opts.failOnMissingDependencies,
+        });
+    }
+    return {
+        ...lockfile,
+        importers,
+        packages,
+    };
+}
+function pkgAllDeps(step, pickedPackages, opts) {
+    for (const { pkgSnapshot, depPath, next } of step.dependencies) {
+        pickedPackages[depPath] = pkgSnapshot;
+        pkgAllDeps(next(), pickedPackages, opts);
+    }
+    for (const depPath of step.missing) {
+        if (opts.failOnMissingDependencies) {
+            throw new LockfileMissingDependencyError(depPath);
+        }
+        lockfileLogger.debug(`No entry for "${depPath}" in ${WANTED_LOCKFILE}`);
+    }
+}
+//# sourceMappingURL=filterLockfileByImporters.js.map

package/lib/filterLockfileByImportersAndEngine.d.ts

@@ -0,0 +1,35 @@
+import type { LockfileObject } from '@pnpm/lockfile.types';
+import type { DependenciesField, DepPath, ProjectId, SupportedArchitectures } from '@pnpm/types';
+export interface FilterLockfileResult {
+    lockfile: LockfileObject;
+    selectedImporterIds: ProjectId[];
+    /**
+     * Dep paths reached by a non-optional edge from an importer or from a
+     * package that is itself part of the install. Their installability is
+     * evaluated as non-optional — an incompatible one fails the install under
+     * `engineStrict` instead of being skipped — even when the lockfile marks
+     * the snapshot `optional: true` because the subtree happens to hang off an
+     * optional dependency. Downstream consumers must classify by this set
+     * rather than by `pkgSnapshot.optional` so every stage of a headless
+     * install agrees with the resolver, which classifies by edge too.
+     */
+    requiredDepPaths: Set<DepPath>;
+}
+export declare function filterLockfileByEngine(lockfile: LockfileObject, opts: FilterLockfileOptions): FilterLockfileResult;
+export interface FilterLockfileOptions {
+    currentEngine: {
+        nodeVersion?: string;
+        pnpmVersion: string;
+    };
+    engineStrict: boolean;
+    include: {
+        [dependenciesField in DependenciesField]: boolean;
+    };
+    includeIncompatiblePackages?: boolean;
+    failOnMissingDependencies: boolean;
+    lockfileDir: string;
+    skipped: Set<string>;
+    skipRuntimes?: boolean;
+    supportedArchitectures?: SupportedArchitectures;
+}
+export declare function filterLockfileByImportersAndEngine(lockfile: LockfileObject, importerIds: ProjectId[], opts: FilterLockfileOptions): FilterLockfileResult;

package/lib/filterLockfileByImportersAndEngine.js

@@ -0,0 +1,266 @@
+import { checkPackageInstallability, packageIsInstallable, } from '@pnpm/config.package-is-installable';
+import { WANTED_LOCKFILE } from '@pnpm/constants';
+import * as dp from '@pnpm/deps.path';
+import { LockfileMissingDependencyError } from '@pnpm/error';
+import { nameVerFromPkgSnapshot } from '@pnpm/lockfile.utils';
+import { logger } from '@pnpm/logger';
+import { map as mapValues, pickBy, unnest } from 'ramda';
+import { filterImporter } from './filterImporter.js';
+const lockfileLogger = logger('lockfile');
+export function filterLockfileByEngine(lockfile, opts) {
+    const importerIds = Object.keys(lockfile.importers);
+    return filterLockfileByImportersAndEngine(lockfile, importerIds, opts);
+}
+export function filterLockfileByImportersAndEngine(lockfile, importerIds, opts) {
+    const importerIdSet = new Set(importerIds);
+    const directDepEdges = toImporterDepPaths(lockfile, importerIds, {
+        include: opts.include,
+        importerIdSet,
+        skipRuntimes: opts.skipRuntimes,
+    });
+    const { packages, requiredDepPaths } = lockfile.packages != null
+        ? pickPkgsWithAllDeps(lockfile, directDepEdges, importerIdSet, {
+            currentEngine: opts.currentEngine,
+            engineStrict: opts.engineStrict,
+            failOnMissingDependencies: opts.failOnMissingDependencies,
+            include: opts.include,
+            includeIncompatiblePackages: opts.includeIncompatiblePackages === true,
+            lockfileDir: opts.lockfileDir,
+            skipped: opts.skipped,
+            skipRuntimes: opts.skipRuntimes,
+            supportedArchitectures: opts.supportedArchitectures,
+        })
+        : { packages: {}, requiredDepPaths: new Set() };
+    const importers = mapValues((importer) => {
+        const newImporter = filterImporter(importer, opts.include, { skipRuntimes: opts.skipRuntimes });
+        if (newImporter.optionalDependencies != null) {
+            newImporter.optionalDependencies = pickBy((ref, depName) => {
+                const depPath = dp.refToRelative(ref, depName);
+                return !depPath || packages[depPath] != null;
+            }, newImporter.optionalDependencies);
+        }
+        return newImporter;
+    }, lockfile.importers);
+    return {
+        lockfile: {
+            ...lockfile,
+            importers,
+            packages,
+        },
+        selectedImporterIds: Array.from(importerIdSet),
+        requiredDepPaths,
+    };
+}
+function pickPkgsWithAllDeps(lockfile, depEdges, importerIdSet, opts) {
+    const ctx = {
+        lockfile,
+        pickedPackages: {},
+        importerIdSet,
+        installed: new Set(),
+        requiredDepPaths: new Set(),
+        evaluated: [],
+        edgesByDepPath: new Map(),
+    };
+    classifyDeps(ctx, depEdges, opts);
+    reportInstallability(ctx, opts);
+    pickSkippedDeps(ctx, depEdges, opts);
+    return { packages: ctx.pickedPackages, requiredDepPaths: ctx.requiredDepPaths };
+}
+/**
+ * Work out which packages the install actually reaches, and over which kind
+ * of edge.
+ *
+ * An edge is only expanded once its source is known to be part of the
+ * install, so the optionality of the edge — not the `optional` flag the
+ * resolver propagated onto the snapshot — decides whether an incompatible
+ * package is skipped: only a package whose every inbound edge is optional
+ * stays out of the install.
+ *
+ * Traversal is breadth-first over edges rather than depth-first over
+ * packages so the verdict does not depend on which edge happens to reach a
+ * package first: a package left out on an optional edge is reconsidered when
+ * a non-optional one arrives later. Nothing is reported from here — a
+ * package can be visited once per inbound edge, and its installability must
+ * be reported exactly once, which [reportInstallability] does afterwards.
+ */
+function classifyDeps(ctx, depEdges, opts) {
+    const incompatible = new Map();
+    // Walked with a cursor rather than `shift()`, which is O(n) per dequeue on
+    // a JS array and would make the walk quadratic on a large lockfile.
+    const queue = [...depEdges];
+    for (let next = 0; next < queue.length; next++) {
+        const { depPath, optional } = queue[next];
+        if (!optional)
+            ctx.requiredDepPaths.add(depPath);
+        if (ctx.installed.has(depPath))
+            continue;
+        const pkgSnapshot = ctx.lockfile.packages[depPath];
+        // Missing entries are reported by the closure pass, which reaches every
+        // dep path this one does.
+        if (!pkgSnapshot)
+            continue;
+        if (!incompatible.has(depPath)) {
+            ctx.evaluated.push(depPath);
+            // TODO: depPath is not the package ID. Should be fixed
+            incompatible.set(depPath, !opts.includeIncompatiblePackages && checkPackageInstallability(pkgSnapshot.id ?? depPath, toInstallabilityManifest(depPath, pkgSnapshot), {
+                nodeVersion: opts.currentEngine.nodeVersion,
+                optional: true,
+                supportedArchitectures: opts.supportedArchitectures,
+            }) != null);
+        }
+        if (optional && incompatible.get(depPath))
+            continue;
+        ctx.installed.add(depPath);
+        ctx.pickedPackages[depPath] = pkgSnapshot;
+        const edges = nextDepEdges(ctx, pkgSnapshot, opts);
+        ctx.edgesByDepPath.set(depPath, edges);
+        // Appended one by one: `push(...edges)` passes each edge as its own
+        // argument and overflows the engine's argument limit on a wide enough
+        // dependency list.
+        for (const edge of edges) {
+            queue.push(edge);
+        }
+    }
+}
+/**
+ * Report each reached package once, against the edge kind it was classified
+ * by. This is where an incompatible package that a non-optional edge reaches
+ * fails the install under `engineStrict`, and where a skipped optional gets
+ * its `skippedOptionalDependencyLogger` entry.
+ */
+function reportInstallability(ctx, opts) {
+    for (const depPath of ctx.evaluated) {
+        const pkgSnapshot = ctx.lockfile.packages[depPath];
+        const installable = opts.includeIncompatiblePackages ||
+            packageIsInstallable(pkgSnapshot.id ?? depPath, toInstallabilityManifest(depPath, pkgSnapshot), {
+                // A subtree hanging off an `optionalDependencies` entry stays
+                // best-effort: the dependency is installed so its dependent is not
+                // left broken, but an incompatibility inside it does not fail the
+                // install. Only a package no optional path reaches is fatal here.
+                engineStrict: opts.engineStrict && pkgSnapshot.optional !== true,
+                lockfileDir: opts.lockfileDir,
+                nodeVersion: opts.currentEngine.nodeVersion,
+                optional: !ctx.installed.has(depPath) || !ctx.requiredDepPaths.has(depPath),
+                supportedArchitectures: opts.supportedArchitectures,
+            }) !== false;
+        if (installable) {
+            opts.skipped.delete(depPath);
+        }
+        else {
+            opts.skipped.add(depPath);
+        }
+    }
+}
+function toInstallabilityManifest(depPath, pkgSnapshot) {
+    return {
+        ...nameVerFromPkgSnapshot(depPath, pkgSnapshot),
+        cpu: pkgSnapshot.cpu,
+        engines: pkgSnapshot.engines,
+        os: pkgSnapshot.os,
+        libc: pkgSnapshot.libc,
+    };
+}
+/**
+ * Extend the picked set over the packages that are reachable but not part of
+ * the install — an incompatible optional and everything below it. They stay
+ * in the filtered lockfile and are recorded as skipped, so a later install on
+ * a host that does support them can pick them up without a re-resolution.
+ */
+function pickSkippedDeps(ctx, depEdges, opts) {
+    const queue = depEdges.map(({ depPath }) => depPath);
+    const visited = new Set();
+    for (let next = 0; next < queue.length; next++) {
+        const depPath = queue[next];
+        if (visited.has(depPath))
+            continue;
+        visited.add(depPath);
+        const pkgSnapshot = ctx.lockfile.packages[depPath];
+        if (!pkgSnapshot && !depPath.startsWith('link:')) {
+            if (opts.failOnMissingDependencies) {
+                throw new LockfileMissingDependencyError(depPath);
+            }
+            lockfileLogger.debug(`No entry for "${depPath}" in ${WANTED_LOCKFILE}`);
+            continue;
+        }
+        if (!ctx.installed.has(depPath)) {
+            if (!ctx.pickedPackages[depPath] && pkgSnapshot.optional === true) {
+                opts.skipped.add(depPath);
+            }
+            ctx.pickedPackages[depPath] = pkgSnapshot;
+        }
+        // `visited` guarantees one pass per dep path, so a cached entry is
+        // released as soon as it is consumed rather than being retained until the
+        // whole walk ends.
+        const edges = ctx.edgesByDepPath.get(depPath) ?? nextDepEdges(ctx, pkgSnapshot, opts);
+        ctx.edgesByDepPath.delete(depPath);
+        for (const edge of edges) {
+            queue.push(edge.depPath);
+        }
+    }
+}
+/** The outbound edges of a package, tagged with the optionality of each. */
+function nextDepEdges(ctx, pkgSnapshot, opts) {
+    const { depEdges, importerIds } = parseDepRefs([
+        ...toDepRefs(pkgSnapshot.dependencies, false),
+        ...(opts.include.optionalDependencies ? toDepRefs(pkgSnapshot.optionalDependencies, true) : []),
+    ], ctx.lockfile);
+    for (const importerId of importerIds) {
+        ctx.importerIdSet.add(importerId);
+    }
+    return [
+        ...depEdges,
+        ...toImporterDepPaths(ctx.lockfile, importerIds, {
+            include: opts.include,
+            importerIdSet: ctx.importerIdSet,
+            skipRuntimes: opts.skipRuntimes,
+        }),
+    ];
+}
+function toImporterDepPaths(lockfile, importerIds, opts) {
+    const importerDeps = importerIds
+        .map(importerId => lockfile.importers[importerId])
+        .map(importer => [
+        ...(opts.include.dependencies ? toDepRefs(importer.dependencies, false) : []),
+        ...(opts.include.devDependencies ? toDepRefs(importer.devDependencies, false) : []),
+        ...(opts.include.optionalDependencies ? toDepRefs(importer.optionalDependencies, true) : []),
+    ])
+        .map(refs => opts.skipRuntimes ? refs.filter(({ ref }) => !ref.startsWith('runtime:')) : refs);
+    let { depEdges, importerIds: nextImporterIds } = parseDepRefs(unnest(importerDeps), lockfile);
+    if (!nextImporterIds.length) {
+        return depEdges;
+    }
+    nextImporterIds = nextImporterIds.filter(importerId => !opts.importerIdSet.has(importerId));
+    for (const importerId of nextImporterIds) {
+        opts.importerIdSet.add(importerId);
+    }
+    return [
+        ...depEdges,
+        ...toImporterDepPaths(lockfile, nextImporterIds, opts),
+    ];
+}
+function toDepRefs(refsByPkgNames, optional) {
+    if (refsByPkgNames == null)
+        return [];
+    return Object.entries(refsByPkgNames).map(([pkgName, ref]) => ({ pkgName, ref, optional }));
+}
+function parseDepRefs(depRefs, lockfile) {
+    const acc = {
+        depEdges: [],
+        importerIds: [],
+    };
+    for (const { pkgName, ref, optional } of depRefs) {
+        if (ref.startsWith('link:')) {
+            const importerId = ref.substring(5);
+            if (lockfile.importers[importerId]) {
+                acc.importerIds.push(importerId);
+            }
+            continue;
+        }
+        const depPath = dp.refToRelative(ref, pkgName);
+        if (depPath == null)
+            continue;
+        acc.depEdges.push({ depPath, optional });
+    }
+    return acc;
+}
+//# sourceMappingURL=filterLockfileByImportersAndEngine.js.map

package/lib/index.d.ts

@@ -0,0 +1,3 @@
+export { filterLockfile } from './filterLockfile.js';
+export { filterLockfileByImporters } from './filterLockfileByImporters.js';
+export { filterLockfileByEngine, filterLockfileByImportersAndEngine } from './filterLockfileByImportersAndEngine.js';

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pnpm/lockfile.filtering",
-  "version": "1100.1.13",
+  "version": "1100.2.0",
   "description": "Filters a lockfile",
   "keywords": [
     "pnpm",
@@ -29,14 +29,14 @@
     "!*.map"
   ],
   "dependencies": {
-    "@pnpm/config.package-is-installable": "1100.0.15",
-    "@pnpm/constants": "1100.0.0",
-    "@pnpm/deps.path": "1100.0.10",
-    "@pnpm/error": "1100.0.1",
-    "@pnpm/lockfile.types": "1100.0.15",
-    "@pnpm/lockfile.utils": "1100.1.4",
-    "@pnpm/lockfile.walker": "1100.0.15",
-    "@pnpm/types": "1101.5.0",
+    "@pnpm/config.package-is-installable": "1100.1.0",
+    "@pnpm/constants": "1100.0.1",
+    "@pnpm/deps.path": "1100.0.12",
+    "@pnpm/error": "1100.1.0",
+    "@pnpm/lockfile.types": "1100.0.17",
+    "@pnpm/lockfile.utils": "1100.1.6",
+    "@pnpm/lockfile.walker": "1100.0.17",
+    "@pnpm/types": "1101.7.0",
     "ramda": "npm:@pnpm/ramda@0.28.1"
   },
   "peerDependencies": {
@@ -44,9 +44,9 @@
   },
   "devDependencies": {
     "@jest/globals": "30.4.1",
-    "@pnpm/lockfile.filtering": "1100.1.13",
+    "@pnpm/lockfile.filtering": "1100.2.0",
     "@pnpm/logger": "1100.0.0",
-    "@types/ramda": "0.31.1",
+    "@types/ramda": "0.32.0",
     "detect-libc": "^2.1.2",
     "tempy": "3.0.0",
     "write-yaml-file": "^6.0.0",