@pnpm/installing.deps-restorer 1006.0.0

package/lib/linkHoistedModules.d.ts

@@ -0,0 +1,14 @@
+import type { DependenciesGraph, DepHierarchy } from '@pnpm/deps.graph-builder';
+import { type DepsStateCache } from '@pnpm/deps.graph-hasher';
+import type { StoreController } from '@pnpm/store.controller-types';
+import type { AllowBuild } from '@pnpm/types';
+export declare function linkHoistedModules(storeController: StoreController, graph: DependenciesGraph, prevGraph: DependenciesGraph, hierarchy: DepHierarchy, opts: {
+    allowBuild?: AllowBuild;
+    depsStateCache: DepsStateCache;
+    disableRelinkLocalDirDeps?: boolean;
+    force: boolean;
+    ignoreScripts: boolean;
+    lockfileDir: string;
+    preferSymlinkedExecutables?: boolean;
+    sideEffectsCacheRead: boolean;
+}): Promise<void>;

package/lib/linkHoistedModules.js

@@ -0,0 +1,106 @@
+import path from 'node:path';
+import { linkBins } from '@pnpm/bins.linker';
+import { progressLogger, removalLogger, statsLogger, } from '@pnpm/core-loggers';
+import { calcDepState } from '@pnpm/deps.graph-hasher';
+import { logger } from '@pnpm/logger';
+import { rimraf } from '@zkochan/rimraf';
+import pLimit from 'p-limit';
+import { difference, isEmpty } from 'ramda';
+const limitLinking = pLimit(16);
+export async function linkHoistedModules(storeController, graph, prevGraph, hierarchy, opts) {
+    // TODO: remove nested node modules first
+    const dirsToRemove = difference(Object.keys(prevGraph), Object.keys(graph));
+    statsLogger.debug({
+        prefix: opts.lockfileDir,
+        removed: dirsToRemove.length,
+    });
+    // We should avoid removing unnecessary directories while simultaneously adding new ones.
+    // Doing so can sometimes lead to a race condition when linking commands to `node_modules/.bin`.
+    await Promise.all(dirsToRemove.map((dir) => tryRemoveDir(dir)));
+    await Promise.all(Object.entries(hierarchy)
+        .map(([parentDir, depsHierarchy]) => {
+        function warn(message) {
+            logger.info({
+                message,
+                prefix: parentDir,
+            });
+        }
+        return linkAllPkgsInOrder(storeController, graph, depsHierarchy, parentDir, {
+            ...opts,
+            warn,
+        });
+    }));
+}
+async function tryRemoveDir(dir) {
+    removalLogger.debug(dir);
+    try {
+        await rimraf(dir);
+    }
+    catch (err) { // eslint-disable-line
+        /* Just ignoring for now. Not even logging.
+        logger.warn({
+          error: err,
+          message: `Failed to remove "${pathToRemove}"`,
+          prefix: lockfileDir,
+        })
+        */
+    }
+}
+async function linkAllPkgsInOrder(storeController, graph, hierarchy, parentDir, opts) {
+    const _calcDepState = calcDepState.bind(null, graph, opts.depsStateCache);
+    await Promise.all(Object.entries(hierarchy).map(async ([dir, deps]) => {
+        const depNode = graph[dir];
+        if (depNode.fetching) {
+            let filesResponse;
+            try {
+                filesResponse = (await depNode.fetching()).files;
+            }
+            catch (err) { // eslint-disable-line
+                if (depNode.optional)
+                    return;
+                throw err;
+            }
+            depNode.requiresBuild = filesResponse.requiresBuild;
+            let sideEffectsCacheKey;
+            if (opts.sideEffectsCacheRead && filesResponse.sideEffectsMaps && !isEmpty(filesResponse.sideEffectsMaps)) {
+                if (opts?.allowBuild?.(depNode.name, depNode.version) !== false) {
+                    sideEffectsCacheKey = _calcDepState(dir, {
+                        includeDepGraphHash: !opts.ignoreScripts && depNode.requiresBuild, // true when is built
+                        patchFileHash: depNode.patch?.hash,
+                    });
+                }
+            }
+            // Limiting the concurrency here fixes an out of memory error.
+            // It is not clear why it helps as importing is also limited inside fs.indexed-pkg-importer.
+            // The out of memory error was reproduced on the teambit/bit repository with the "rootComponents" feature turned on
+            await limitLinking(async () => {
+                const { importMethod, isBuilt } = await storeController.importPackage(depNode.dir, {
+                    filesResponse,
+                    force: true,
+                    disableRelinkLocalDirDeps: opts.disableRelinkLocalDirDeps,
+                    keepModulesDir: true,
+                    requiresBuild: depNode.patch != null || depNode.requiresBuild,
+                    sideEffectsCacheKey,
+                });
+                if (importMethod) {
+                    progressLogger.debug({
+                        method: importMethod,
+                        requester: opts.lockfileDir,
+                        status: 'imported',
+                        to: depNode.dir,
+                    });
+                }
+                depNode.isBuilt = isBuilt;
+            });
+        }
+        return linkAllPkgsInOrder(storeController, graph, deps, dir, opts);
+    }));
+    const modulesDir = path.join(parentDir, 'node_modules');
+    const binsDir = path.join(modulesDir, '.bin');
+    await linkBins(modulesDir, binsDir, {
+        allowExoticManifests: true,
+        preferSymlinkedExecutables: opts.preferSymlinkedExecutables,
+        warn: opts.warn,
+    });
+}
+//# sourceMappingURL=linkHoistedModules.js.map

package/lib/lockfileToHoistedDepGraph.d.ts

@@ -0,0 +1,38 @@
+import type { LockfileToDepGraphResult } from '@pnpm/deps.graph-builder';
+import { type HoistingLimits } from '@pnpm/installing.linking.real-hoist';
+import type { IncludedDependencies } from '@pnpm/installing.modules-yaml';
+import type { LockfileObject } from '@pnpm/lockfile.fs';
+import { type PatchGroupRecord } from '@pnpm/patching.config';
+import type { StoreController } from '@pnpm/store.controller-types';
+import type { AllowBuild, Registries, SupportedArchitectures } from '@pnpm/types';
+export interface LockfileToHoistedDepGraphOptions {
+    allowBuild?: AllowBuild;
+    autoInstallPeers: boolean;
+    engineStrict: boolean;
+    force: boolean;
+    hoistingLimits?: HoistingLimits;
+    externalDependencies?: Set<string>;
+    importerIds: string[];
+    include: IncludedDependencies;
+    ignoreScripts: boolean;
+    /**
+     * When true, skip fetching local dependencies (file: protocol pointing to directories).
+     * This is used by `pnpm fetch` which only downloads packages from the registry
+     * and doesn't need local packages that won't be available (e.g., in Docker builds).
+     */
+    ignoreLocalPackages?: boolean;
+    currentHoistedLocations?: Record<string, string[]>;
+    lockfileDir: string;
+    modulesDir?: string;
+    nodeVersion: string;
+    pnpmVersion: string;
+    registries: Registries;
+    patchedDependencies?: PatchGroupRecord;
+    sideEffectsCacheRead: boolean;
+    skipped: Set<string>;
+    storeController: StoreController;
+    storeDir: string;
+    virtualStoreDir: string;
+    supportedArchitectures?: SupportedArchitectures;
+}
+export declare function lockfileToHoistedDepGraph(lockfile: LockfileObject, currentLockfile: LockfileObject | null, opts: LockfileToHoistedDepGraphOptions): Promise<LockfileToDepGraphResult>;

package/lib/lockfileToHoistedDepGraph.js

@@ -0,0 +1,251 @@
+import path from 'node:path';
+import { packageIsInstallable } from '@pnpm/config.package-is-installable';
+import * as dp from '@pnpm/deps.path';
+import { hoist } from '@pnpm/installing.linking.real-hoist';
+import { nameVerFromPkgSnapshot, packageIdFromSnapshot, pkgSnapshotToResolution, } from '@pnpm/lockfile.utils';
+import { logger } from '@pnpm/logger';
+import { getPatchInfo } from '@pnpm/patching.config';
+import { safeReadPackageJsonFromDir } from '@pnpm/pkg-manifest.reader';
+import { pathExists } from 'path-exists';
+export async function lockfileToHoistedDepGraph(lockfile, currentLockfile, opts) {
+    let prevGraph;
+    if (currentLockfile?.packages != null) {
+        prevGraph = (await _lockfileToHoistedDepGraph(currentLockfile, {
+            ...opts,
+            force: true,
+            skipped: new Set(),
+        })).graph;
+    }
+    else {
+        prevGraph = {};
+    }
+    return {
+        ...(await _lockfileToHoistedDepGraph(lockfile, opts)),
+        prevGraph,
+    };
+}
+async function _lockfileToHoistedDepGraph(lockfile, opts) {
+    const tree = hoist(lockfile, {
+        hoistingLimits: opts.hoistingLimits,
+        externalDependencies: opts.externalDependencies,
+        autoInstallPeers: opts.autoInstallPeers,
+    });
+    const graph = {};
+    const modulesDir = path.join(opts.lockfileDir, opts.modulesDir ?? 'node_modules');
+    const fetchDepsOpts = {
+        ...opts,
+        lockfile,
+        graph,
+        pkgLocationsByDepPath: {},
+        injectionTargetsByDepPath: new Map(),
+        hoistedLocations: {},
+    };
+    const hierarchy = {
+        [opts.lockfileDir]: await fetchDeps(fetchDepsOpts, modulesDir, tree.dependencies),
+    };
+    const directDependenciesByImporterId = {
+        '.': directDepsMap(Object.keys(hierarchy[opts.lockfileDir]), graph),
+    };
+    const symlinkedDirectDependenciesByImporterId = { '.': {} };
+    await Promise.all(Array.from(tree.dependencies).map(async (rootDep) => {
+        const reference = Array.from(rootDep.references)[0];
+        if (reference.startsWith('workspace:')) {
+            const importerId = reference.replace('workspace:', '');
+            const projectDir = path.join(opts.lockfileDir, importerId);
+            const modulesDir = path.join(projectDir, 'node_modules');
+            const nextHierarchy = (await fetchDeps(fetchDepsOpts, modulesDir, rootDep.dependencies));
+            hierarchy[projectDir] = nextHierarchy;
+            const importer = lockfile.importers[importerId];
+            const importerDir = path.join(opts.lockfileDir, importerId);
+            symlinkedDirectDependenciesByImporterId[importerId] = pickLinkedDirectDeps(importer, importerDir, opts.include);
+            directDependenciesByImporterId[importerId] = directDepsMap(Object.keys(nextHierarchy), graph);
+        }
+    }));
+    return {
+        directDependenciesByImporterId,
+        graph,
+        hierarchy,
+        symlinkedDirectDependenciesByImporterId,
+        hoistedLocations: fetchDepsOpts.hoistedLocations,
+        injectionTargetsByDepPath: fetchDepsOpts.injectionTargetsByDepPath,
+    };
+}
+function directDepsMap(directDepDirs, graph) {
+    const acc = {};
+    for (const dir of directDepDirs) {
+        acc[graph[dir].alias] = dir;
+    }
+    return acc;
+}
+function pickLinkedDirectDeps(importer, importerDir, include) {
+    const rootDeps = {
+        ...(include.devDependencies ? importer.devDependencies : {}),
+        ...(include.dependencies ? importer.dependencies : {}),
+        ...(include.optionalDependencies ? importer.optionalDependencies : {}),
+    };
+    const directDeps = {};
+    for (const alias in rootDeps) {
+        const ref = rootDeps[alias];
+        if (ref.startsWith('link:')) {
+            directDeps[alias] = path.resolve(importerDir, ref.slice(5));
+        }
+    }
+    return directDeps;
+}
+async function fetchDeps(opts, modules, deps) {
+    const depHierarchy = {};
+    await Promise.all(Array.from(deps).map(async (dep) => {
+        const depPath = Array.from(dep.references)[0];
+        if (opts.skipped.has(depPath) || depPath.startsWith('workspace:'))
+            return;
+        const pkgSnapshot = opts.lockfile.packages[depPath];
+        if (!pkgSnapshot) {
+            // it is a link
+            return;
+        }
+        const { name: pkgName, version: pkgVersion } = nameVerFromPkgSnapshot(depPath, pkgSnapshot);
+        const packageId = packageIdFromSnapshot(depPath, pkgSnapshot);
+        const pkgIdWithPatchHash = dp.getPkgIdWithPatchHash(depPath);
+        const pkg = {
+            name: pkgName,
+            version: pkgVersion,
+            engines: pkgSnapshot.engines,
+            cpu: pkgSnapshot.cpu,
+            os: pkgSnapshot.os,
+            libc: pkgSnapshot.libc,
+        };
+        if (!opts.force &&
+            packageIsInstallable(packageId, pkg, {
+                engineStrict: opts.engineStrict,
+                lockfileDir: opts.lockfileDir,
+                nodeVersion: opts.nodeVersion,
+                optional: pkgSnapshot.optional === true,
+                supportedArchitectures: opts.supportedArchitectures,
+            }) === false) {
+            opts.skipped.add(depPath);
+            return;
+        }
+        const isDirectoryDep = 'directory' in pkgSnapshot.resolution && pkgSnapshot.resolution.directory != null;
+        if (isDirectoryDep && opts.ignoreLocalPackages) {
+            logger.info({
+                message: `Skipping local dependency ${pkgName}@${pkgVersion} (file: protocol)`,
+                prefix: opts.lockfileDir,
+            });
+            return;
+        }
+        const dir = path.join(modules, dep.name);
+        const depLocation = path.relative(opts.lockfileDir, dir);
+        const resolution = pkgSnapshotToResolution(depPath, pkgSnapshot, opts.registries);
+        let fetchResponse;
+        // We check for the existence of the package inside node_modules.
+        // It will only be missing if the user manually removed it.
+        // That shouldn't normally happen but Bit CLI does remove node_modules in component directories:
+        // https://github.com/teambit/bit/blob/5e1eed7cd122813ad5ea124df956ee89d661d770/scopes/dependencies/dependency-resolver/dependency-installer.ts#L169
+        //
+        // We also verify that the package that is present has the expected version.
+        // This check is required because there is no guarantee the modules manifest and current lockfile were
+        // successfully saved after node_modules was changed during installation.
+        const skipFetch = opts.currentHoistedLocations?.[depPath]?.includes(depLocation) &&
+            await dirHasPackageJsonWithVersion(path.join(opts.lockfileDir, depLocation), pkgVersion);
+        const pkgResolution = {
+            id: packageId,
+            resolution,
+            name: pkgName,
+            version: pkgVersion,
+        };
+        if (skipFetch) {
+            const { filesIndexFile } = opts.storeController.getFilesIndexFilePath({
+                ignoreScripts: opts.ignoreScripts,
+                pkg: pkgResolution,
+            });
+            fetchResponse = { filesIndexFile };
+        }
+        else {
+            try {
+                fetchResponse = opts.storeController.fetchPackage({
+                    allowBuild: opts.allowBuild,
+                    force: false,
+                    lockfileDir: opts.lockfileDir,
+                    ignoreScripts: opts.ignoreScripts,
+                    pkg: pkgResolution,
+                    supportedArchitectures: opts.supportedArchitectures,
+                });
+                if (fetchResponse instanceof Promise)
+                    fetchResponse = await fetchResponse;
+            }
+            catch (err) {
+                if (pkgSnapshot.optional)
+                    return;
+                throw err;
+            }
+        }
+        opts.graph[dir] = {
+            alias: dep.name,
+            children: {},
+            depPath,
+            pkgIdWithPatchHash,
+            dir,
+            fetching: fetchResponse.fetching,
+            filesIndexFile: fetchResponse.filesIndexFile,
+            hasBin: pkgSnapshot.hasBin === true,
+            hasBundledDependencies: pkgSnapshot.bundledDependencies != null,
+            modules,
+            name: pkgName,
+            version: pkgVersion,
+            optional: !!pkgSnapshot.optional,
+            optionalDependencies: new Set(Object.keys(pkgSnapshot.optionalDependencies ?? {})),
+            patch: getPatchInfo(opts.patchedDependencies, pkgName, pkgVersion),
+            resolution: pkgSnapshot.resolution,
+        };
+        if (!opts.pkgLocationsByDepPath[depPath]) {
+            opts.pkgLocationsByDepPath[depPath] = [];
+        }
+        opts.pkgLocationsByDepPath[depPath].push(dir);
+        // Track directory deps for injected workspace packages
+        if ('directory' in pkgSnapshot.resolution && pkgSnapshot.resolution.directory != null) {
+            const locations = opts.injectionTargetsByDepPath.get(depPath);
+            if (locations) {
+                locations.push(dir);
+            }
+            else {
+                opts.injectionTargetsByDepPath.set(depPath, [dir]);
+            }
+        }
+        depHierarchy[dir] = await fetchDeps(opts, path.join(dir, 'node_modules'), dep.dependencies);
+        if (!opts.hoistedLocations[depPath]) {
+            opts.hoistedLocations[depPath] = [];
+        }
+        opts.hoistedLocations[depPath].push(depLocation);
+        opts.graph[dir].children = getChildren(pkgSnapshot, opts.pkgLocationsByDepPath, opts);
+    }));
+    return depHierarchy;
+}
+async function dirHasPackageJsonWithVersion(dir, expectedVersion) {
+    if (!expectedVersion)
+        return pathExists(dir);
+    try {
+        const manifest = await safeReadPackageJsonFromDir(dir);
+        return manifest?.version === expectedVersion;
+    }
+    catch (err) {
+        if (err?.code === 'ENOENT') {
+            return pathExists(dir);
+        }
+        throw err;
+    }
+}
+function getChildren(pkgSnapshot, pkgLocationsByDepPath, opts) {
+    const allDeps = {
+        ...pkgSnapshot.dependencies,
+        ...(opts.include.optionalDependencies ? pkgSnapshot.optionalDependencies : {}),
+    };
+    const children = {};
+    for (const [childName, childRef] of Object.entries(allDeps)) {
+        const childDepPath = dp.refToRelative(childRef, childName);
+        if (childDepPath && pkgLocationsByDepPath[childDepPath]) {
+            children[childName] = pkgLocationsByDepPath[childDepPath][0];
+        }
+    }
+    return children;
+}
+//# sourceMappingURL=lockfileToHoistedDepGraph.js.map

package/package.json

@@ -0,0 +1,108 @@
+{
+  "name": "@pnpm/installing.deps-restorer",
+  "version": "1006.0.0",
+  "description": "Fast installation using only pnpm-lock.yaml",
+  "keywords": [
+    "pnpm",
+    "pnpm11",
+    "install",
+    "installer"
+  ],
+  "license": "MIT",
+  "funding": "https://opencollective.com/pnpm",
+  "repository": "https://github.com/pnpm/pnpm/tree/main/installing/deps-restorer",
+  "homepage": "https://github.com/pnpm/pnpm/tree/main/installing/deps-restorer#readme",
+  "bugs": {
+    "url": "https://github.com/pnpm/pnpm/issues"
+  },
+  "type": "module",
+  "main": "lib/index.js",
+  "types": "lib/index.d.ts",
+  "exports": {
+    ".": "./lib/index.js"
+  },
+  "files": [
+    "lib",
+    "!*.map"
+  ],
+  "directories": {
+    "test": "test"
+  },
+  "dependencies": {
+    "@zkochan/rimraf": "^4.0.0",
+    "p-limit": "^7.1.0",
+    "path-absolute": "^2.0.0",
+    "path-exists": "^5.0.0",
+    "ramda": "npm:@pnpm/ramda@0.28.1",
+    "realpath-missing": "^2.0.0",
+    "@pnpm/bins.linker": "1000.2.6",
+    "@pnpm/building.policy": "1000.0.0-0",
+    "@pnpm/building.during-install": "1000.0.0-0",
+    "@pnpm/config.package-is-installable": "1000.0.15",
+    "@pnpm/constants": "1001.3.1",
+    "@pnpm/deps.graph-builder": "1002.3.0",
+    "@pnpm/core-loggers": "1001.0.4",
+    "@pnpm/deps.graph-hasher": "1002.0.8",
+    "@pnpm/error": "1000.0.5",
+    "@pnpm/exec.lifecycle": "1001.0.25",
+    "@pnpm/fs.symlink-dependency": "1000.0.12",
+    "@pnpm/deps.path": "1001.1.3",
+    "@pnpm/installing.linking.direct-dep-linker": "1000.0.12",
+    "@pnpm/installing.linking.hoist": "1002.0.8",
+    "@pnpm/installing.linking.modules-cleaner": "1001.0.23",
+    "@pnpm/installing.linking.real-hoist": "1001.0.20",
+    "@pnpm/installing.modules-yaml": "1000.3.6",
+    "@pnpm/installing.package-requester": "1008.0.0",
+    "@pnpm/lockfile.filtering": "1001.0.21",
+    "@pnpm/lockfile.utils": "1003.0.3",
+    "@pnpm/lockfile.to-pnp": "1001.0.23",
+    "@pnpm/pkg-manifest.reader": "1000.1.2",
+    "@pnpm/lockfile.fs": "1001.1.21",
+    "@pnpm/store.controller-types": "1004.1.0",
+    "@pnpm/patching.config": "1001.0.11",
+    "@pnpm/types": "1000.9.0",
+    "@pnpm/workspace.project-manifest-reader": "1001.1.4"
+  },
+  "peerDependencies": {
+    "@pnpm/logger": ">=1001.0.0 <1002.0.0",
+    "@pnpm/worker": "^1000.3.0"
+  },
+  "devDependencies": {
+    "@jest/globals": "30.0.5",
+    "@pnpm/registry-mock": "5.2.4",
+    "@types/fs-extra": "^9.0.13",
+    "@types/ramda": "0.29.12",
+    "concurrently": "9.2.1",
+    "isexe": "2.0.0",
+    "load-json-file": "^7.0.1",
+    "tempy": "3.0.0",
+    "write-json-file": "^7.0.0",
+    "@pnpm/installing.deps-restorer": "1006.0.0",
+    "@pnpm/crypto.object-hasher": "1000.1.0",
+    "@pnpm/installing.read-projects-context": "1000.0.24",
+    "@pnpm/assert-project": "1000.0.4",
+    "@pnpm/logger": "1001.0.1",
+    "@pnpm/prepare": "1000.0.4",
+    "@pnpm/store.cafs": "1000.0.19",
+    "@pnpm/store.path": "1000.0.5",
+    "@pnpm/store.index": "1000.0.0-0",
+    "@pnpm/test-fixtures": "1000.0.0",
+    "@pnpm/testing.temp-store": "1000.0.23",
+    "@pnpm/test-ipc-server": "1000.0.0"
+  },
+  "engines": {
+    "node": ">=22.13"
+  },
+  "jest": {
+    "preset": "@pnpm/jest-config/with-registry"
+  },
+  "scripts": {
+    "start": "tsgo --watch",
+    "lint": "eslint \"src/**/*.ts\" \"test/**/*.ts\"",
+    "_test": "cross-env NODE_OPTIONS=\"$NODE_OPTIONS --experimental-vm-modules --disable-warning=ExperimentalWarning --disable-warning=DEP0169\" jest",
+    "test": "pnpm run compile && pnpm run _test",
+    "runPrepareFixtures": "node ../../pnpm/bin/pnpm.mjs i -r -C test/fixtures --no-shared-workspace-lockfile --no-link-workspace-packages --lockfile-only --registry http://localhost:4873/ --ignore-scripts --force --no-strict-peer-dependencies",
+    "prepareFixtures": "git clean -fdx test/fixtures && rm -rf \"test/fixtures/*/pnpm-lock.yaml\" && registry-mock prepare && concurrently --success=first --kill-others registry-mock \"pnpm run runPrepareFixtures\"",
+    "compile": "tsgo --build && pnpm run lint --fix"
+  }
+}