@pnpm/workspace.injected-deps-syncer 1100.0.38 → 1100.0.40

package/CHANGELOG.md

@@ -1,5 +1,47 @@
 # @pnpm/workspace.injected-deps-syncer
 
+## 1100.0.40
+
+### Patch Changes
+
+- `pnpm install` no longer fails for an injected workspace dependency whose package publishes from a `publishConfig.directory` that its own `prepare` script builds. The injected copy now picks up that directory once `prepare` finishes building it. `pnpm install --frozen-lockfile` no longer reports the dependency as outdated while the directory has not been built yet. [pnpm/pnpm#7811](https://github.com/pnpm/pnpm/issues/7811)
+
+- With `sharedWorkspaceLockfile: false`, an injected workspace package that has lifecycle scripts is now hard linked into the projects that depend on it. Before, pnpm left a plain copy, so later edits to the package did not reach those projects [#9828](https://github.com/pnpm/pnpm/issues/9828).
+
+- Scripts listed in `syncInjectedDepsAfterScripts` now update injected dependencies while they run. A watcher on the injected package, such as a dev server, sees each change before the script exits [pnpm/pnpm#4410](https://github.com/pnpm/pnpm/issues/4410).
+
+- `pnpm install` now works in StackBlitz WebContainers on projects without a lockfile. It used to fail there with `ENOENT ... pnpm-lock.yaml`, because pnpm did not recognize the errors that WebContainers return from asynchronous file system calls [#15649](https://github.com/pnpm/pnpm/issues/15649).
+
+- Updated dependencies:
+  - @pnpm/bins.linker@1100.0.34
+  - @pnpm/bins.remover@1100.0.26
+  - @pnpm/error@1100.2.1
+  - @pnpm/fetching.directory-fetcher@1100.0.36
+  - @pnpm/fs.graceful-fs@1100.2.4
+  - @pnpm/installing.modules-yaml@1101.0.5
+  - @pnpm/pkg-manifest.reader@1100.0.21
+  - @pnpm/workspace.projects-reader@1101.1.1
+
+## 1100.0.39
+
+### Patch Changes
+
+- `syncInjectedDepsAfterScripts` now copies files into injected dependencies when `node_modules` is on another filesystem than the package sources. The sync previously failed with a cross-device link error and made the script run exit with an error [pnpm/pnpm#14703](https://github.com/pnpm/pnpm/issues/14703).
+
+- Fixed injected workspace dependency synchronization failing with `EPERM` on Windows when removing nested directories.
+
+- Updated dependencies:
+  - @pnpm/bins.linker@1100.0.33
+  - @pnpm/bins.remover@1100.0.25
+  - @pnpm/bins.resolver@1100.0.17
+  - @pnpm/error@1100.2.0
+  - @pnpm/fetching.directory-fetcher@1100.0.35
+  - @pnpm/fs.graceful-fs@1100.2.3
+  - @pnpm/installing.modules-yaml@1101.0.4
+  - @pnpm/pkg-manifest.reader@1100.0.20
+  - @pnpm/types@1102.1.1
+  - @pnpm/workspace.projects-reader@1101.1.0
+
 ## 1100.0.38
 
 ### Patch Changes

package/lib/DirPatcher.d.ts

@@ -62,6 +62,19 @@ export interface ExtendFilesMapOptions {
  * into an inodes map, which is a map from relative path of every file and directory to their inode type.
  */
 export declare function extendFilesMap({ filesMap, filesStats }: ExtendFilesMapOptions): Promise<InodeMap>;
+/** A source directory's files, read once per poll and published to each of its injected copies. */
+export interface PublishSource {
+    dir: string;
+    map: InodeMap;
+}
+export declare function readPublishSource(dir: string): Promise<PublishSource>;
+/**
+ * Copy changed files into `targetDir` as independent files, so a watcher on
+ * the injected directory sees the write. A hardlink edited in place is
+ * republished when its mtime is at least `editedSinceMs`. A copy whose size
+ * and mtime already match the source is left alone.
+ */
+export declare function publishEditsForWatchers(source: PublishSource, targetDir: string, editedSinceMs: number): Promise<void>;
 export declare class DirPatcher {
     private readonly sourceDir;
     private readonly targetDir;

package/lib/DirPatcher.js

@@ -1,7 +1,10 @@
 import fs from 'node:fs';
 import path from 'node:path';
-import util from 'node:util';
+import { pipeline } from 'node:stream/promises';
+import { isError } from '@pnpm/error';
 import { fetchFromDir } from '@pnpm/fetching.directory-fetcher';
+import { renameFileWithRetry, renameFileWithRetryAsync } from '@pnpm/fs.graceful-fs';
+import { pathTemp } from 'path-temp';
 export const DIR = Symbol('Path is a directory');
 // length comparison should place every directory before the files it contains because
 // a directory path is always shorter than any file path it contains
@@ -39,12 +42,41 @@ export async function applyPatch(optimizedDirPatch, sourceDir, targetDir) {
         }
         else if (typeof value === 'string') {
             fs.mkdirSync(path.dirname(targetPath), { recursive: true });
-            await retryOverBlockingInode(targetPath, async () => fs.promises.link(sourcePath, targetPath));
+            await retryOverBlockingInode(targetPath, async () => linkOrCopy(sourcePath, targetPath));
         }
         else {
             const _ = value; // static type guard
         }
     }
+    async function linkOrCopy(sourcePath, targetPath) {
+        try {
+            await fs.promises.link(sourcePath, targetPath);
+        }
+        catch (error) {
+            if (isError(error) && 'code' in error && error.code === 'EXDEV') {
+                await copyIntoPlace(sourcePath, targetPath);
+                return;
+            }
+            throw error;
+        }
+    }
+    /**
+     * Copy through a temp sibling, so that a reader of the target never sees a
+     * partial copy. The rename replaces a symlink at the target without following
+     * it.
+     */
+    async function copyIntoPlace(sourcePath, targetPath) {
+        // A random name, since concurrent syncs share the thread that writes it.
+        const tempPath = pathTemp(path.dirname(targetPath));
+        try {
+            await fs.promises.copyFile(sourcePath, tempPath, fs.constants.COPYFILE_EXCL);
+            renameFileWithRetry(tempPath, targetPath);
+        }
+        catch (error) {
+            await fs.promises.rm(tempPath, { force: true });
+            throw error;
+        }
+    }
     /**
      * The target may hold an inode that {@link extendFilesMap} skips — a FIFO, a
      * socket, a device. The diff cannot see it, so it is never scheduled for
@@ -56,7 +88,7 @@ export async function applyPatch(optimizedDirPatch, sourceDir, targetDir) {
             await add();
         }
         catch (error) {
-            if (!util.types.isNativeError(error) || !('code' in error) || (error.code !== 'EEXIST')) {
+            if (!isError(error) || !('code' in error) || (error.code !== 'EEXIST')) {
                 throw error;
             }
             await removeRecursive(targetPath);
@@ -68,7 +100,7 @@ export async function applyPatch(optimizedDirPatch, sourceDir, targetDir) {
             await fs.promises.rm(targetPath, { recursive: true, force: true });
         }
         catch (error) {
-            if (!util.types.isNativeError(error) || !('code' in error) || (error.code !== 'ENOENT')) {
+            if (!isError(error) || !('code' in error) || (error.code !== 'ENOENT')) {
                 throw error;
             }
         }
@@ -93,9 +125,9 @@ export async function applyPatch(optimizedDirPatch, sourceDir, targetDir) {
     // files a sibling had already linked. A path the target holds as a file and
     // the source as a directory lands in `modified` rather than `added`, so both
     // arrays feed the directory pass.
-    await Promise.all(optimizedDirPatch.removed.map(async (item) => {
-        await removeRecursive(path.join(targetDir, item.path));
-    }));
+    for (const item of optimizedDirPatch.removed) {
+        await removeRecursive(path.join(targetDir, item.path)); // eslint-disable-line no-await-in-loop
+    }
     for (const item of newDirs) {
         await applyChange(item); // eslint-disable-line no-await-in-loop
     }
@@ -130,6 +162,105 @@ export async function extendFilesMap({ filesMap, filesStats }) {
     return result;
 }
 const fileId = (stats) => `${stats.dev}:${stats.ino}`;
+const WATCH_MTIME_TOLERANCE_MS = 1;
+const WATCH_FETCH_OPTIONS = { resolveSymlinks: false };
+export async function readPublishSource(dir) {
+    return { dir, map: await extendFilesMap(await fetchFromDir(dir, WATCH_FETCH_OPTIONS)) };
+}
+/**
+ * Copy changed files into `targetDir` as independent files, so a watcher on
+ * the injected directory sees the write. A hardlink edited in place is
+ * republished when its mtime is at least `editedSinceMs`. A copy whose size
+ * and mtime already match the source is left alone.
+ */
+export async function publishEditsForWatchers(source, targetDir, editedSinceMs) {
+    const { dir: sourceDir, map: sourceMap } = source;
+    const targetMap = await extendFilesMap(await fetchFromDir(targetDir, WATCH_FETCH_OPTIONS));
+    const removed = Object.keys(targetMap)
+        .filter(relPath => !(relPath in sourceMap) && relPath !== '.')
+        .sort(comparePaths)
+        .reverse();
+    for (const relPath of removed) {
+        await removePath(path.join(targetDir, relPath)); // eslint-disable-line no-await-in-loop
+    }
+    const sourcePaths = Object.keys(sourceMap).sort(comparePaths);
+    for (const relPath of sourcePaths) {
+        if (sourceMap[relPath] !== DIR || relPath === '.')
+            continue;
+        const targetPath = path.join(targetDir, relPath);
+        if (targetMap[relPath] != null && targetMap[relPath] !== DIR) {
+            await removePath(targetPath); // eslint-disable-line no-await-in-loop
+        }
+        await fs.promises.mkdir(targetPath, { recursive: true }); // eslint-disable-line no-await-in-loop
+    }
+    for (const relPath of sourcePaths) {
+        const sourceValue = sourceMap[relPath];
+        if (typeof sourceValue !== 'string')
+            continue;
+        const sourcePath = path.join(sourceDir, relPath);
+        const targetPath = path.join(targetDir, relPath);
+        const sourceStat = await fs.promises.stat(sourcePath); // eslint-disable-line no-await-in-loop
+        const targetValue = targetMap[relPath];
+        const targetStat = typeof targetValue === 'string'
+            ? await statFile(targetPath) // eslint-disable-line no-await-in-loop
+            : null;
+        if (!shouldPublish({ sourceStat, targetStat, sourceId: sourceValue, targetValue, editedSinceMs }))
+            continue;
+        await copyForWatchers(sourcePath, targetPath, sourceStat); // eslint-disable-line no-await-in-loop
+    }
+}
+function shouldPublish({ sourceStat, targetStat, sourceId, targetValue, editedSinceMs }) {
+    if (targetStat == null || typeof targetValue !== 'string')
+        return true;
+    if (targetValue === sourceId)
+        return sourceStat.mtimeMs >= editedSinceMs;
+    return sourceStat.size !== targetStat.size ||
+        Math.abs(sourceStat.mtimeMs - targetStat.mtimeMs) > WATCH_MTIME_TOLERANCE_MS;
+}
+async function statFile(filePath) {
+    try {
+        return await fs.promises.stat(filePath);
+    }
+    catch (error) {
+        if (isEnoent(error))
+            return null;
+        throw error;
+    }
+}
+function isEnoent(error) {
+    return isError(error) && 'code' in error && error.code === 'ENOENT';
+}
+async function removePath(targetPath) {
+    await fs.promises.rm(targetPath, { recursive: true, force: true });
+}
+/**
+ * Read and write the bytes. `fs.copyFile` may reflink on macOS, and a
+ * reflink does not notify watchers the way a new file in the injected
+ * directory does.
+ */
+async function copyForWatchers(sourcePath, targetPath, sourceStat) {
+    const existing = await fs.promises.lstat(targetPath).catch((error) => {
+        if (isEnoent(error))
+            return null;
+        throw error;
+    });
+    if (existing?.isDirectory() === true) {
+        await removePath(targetPath);
+    }
+    await fs.promises.mkdir(path.dirname(targetPath), { recursive: true });
+    const tempPath = pathTemp(path.dirname(targetPath));
+    try {
+        await pipeline(fs.createReadStream(sourcePath), fs.createWriteStream(tempPath, { flags: 'wx' }));
+        await fs.promises.chmod(tempPath, sourceStat.mode & 0o7777);
+        await renameFileWithRetryAsync(tempPath, targetPath);
+    }
+    catch (error) {
+        await fs.promises.rm(tempPath, { force: true });
+        throw error;
+    }
+    // Best effort: a target left with a different mtime is only published again on the next poll.
+    await fs.promises.utimes(targetPath, new Date(), sourceStat.mtime).catch(() => { });
+}
 export class DirPatcher {
     sourceDir;
     targetDir;

package/lib/index.d.ts

@@ -11,4 +11,39 @@ export interface SyncInjectedDepsOptions {
      */
     manifestBeforeScripts?: DependencyManifest;
 }
+export interface InjectedEditWatch {
+    stop: () => Promise<void>;
+}
+/**
+ * The source directory and injected copies a running script should publish.
+ * `undefined` when the package has no name, no workspace, or no injected copies,
+ * or when the modules manifest cannot be read. The sync after the script
+ * reports that error, so it does not stop the script from starting.
+ */
+export declare function injectedEditDirs(opts: SyncInjectedDepsOptions): Promise<{
+    sourceDir: string;
+    targetDirs: string[];
+} | undefined>;
+/**
+ * Publish injected copies about every 200ms until `stop`. `editedSinceMs` stays
+ * two seconds behind the start of the watch, which covers one-second mtime
+ * resolution. Stop waits for an in-flight publish so the end-of-script hardlink
+ * sync does not run beside it.
+ */
+export declare function watchInjectedEdits(sourceDir: string, targetDirs: string[]): InjectedEditWatch;
 export declare function syncInjectedDeps(opts: SyncInjectedDepsOptions): Promise<void>;
+export interface SyncInjectedDepsOfModulesDirOptions {
+    /** The directory whose lockfile lists the injected dependencies. */
+    lockfileDir: string;
+    /** The modules directory that holds the `.modules.yaml` of `lockfileDir`. */
+    modulesDir: string;
+    /** The sources whose copies are synced. Copies of other sources keep their contents. */
+    sourceDirs: ReadonlySet<string>;
+}
+/**
+ * Brings the injected copies listed in one modules directory back in step
+ * with their sources. A project with its own lockfile injects copies of
+ * workspace projects whose lifecycle scripts run in their own install, so
+ * nothing else syncs those copies after the scripts run.
+ */
+export declare function syncInjectedDepsOfModulesDir(opts: SyncInjectedDepsOfModulesDirOptions): Promise<void>;

package/lib/index.js

@@ -1,15 +1,94 @@
+import fs from 'node:fs';
 import path from 'node:path';
 import { linkBins, linkBinsOfPackages } from '@pnpm/bins.linker';
 import { removeBin } from '@pnpm/bins.remover';
 import { getBinsFromPackageManifest } from '@pnpm/bins.resolver';
-import { PnpmError } from '@pnpm/error';
+import { isError, PnpmError } from '@pnpm/error';
 import { readModulesManifest } from '@pnpm/installing.modules-yaml';
 import { logger as createLogger } from '@pnpm/logger';
 import { safeReadPackageJsonFromDir } from '@pnpm/pkg-manifest.reader';
 import { findWorkspaceProjectsNoCheck } from '@pnpm/workspace.projects-reader';
 import normalizePath from 'normalize-path';
-import { DirPatcher } from './DirPatcher.js';
+import { DirPatcher, publishEditsForWatchers, readPublishSource } from './DirPatcher.js';
 const logger = createLogger('skip-sync-injected-deps');
+const watchLogger = createLogger('sync-injected-deps-watch');
+/**
+ * The source directory and injected copies a running script should publish.
+ * `undefined` when the package has no name, no workspace, or no injected copies,
+ * or when the modules manifest cannot be read. The sync after the script
+ * reports that error, so it does not stop the script from starting.
+ */
+export async function injectedEditDirs(opts) {
+    if (!opts.pkgName || opts.workspaceDir == null)
+        return undefined;
+    let located;
+    try {
+        located = await readInjectedTargets(opts.workspaceDir, opts.pkgRootDir);
+    }
+    catch (err) {
+        watchLogger.debug({
+            err,
+            message: `Not publishing injected dependencies of ${opts.pkgRootDir} while its script is running`,
+        });
+        return undefined;
+    }
+    if (located?.resolvedTargetDirs == null)
+        return undefined;
+    return { sourceDir: located.pkgRootDir, targetDirs: located.resolvedTargetDirs };
+}
+/**
+ * Publish injected copies about every 200ms until `stop`. `editedSinceMs` stays
+ * two seconds behind the start of the watch, which covers one-second mtime
+ * resolution. Stop waits for an in-flight publish so the end-of-script hardlink
+ * sync does not run beside it.
+ */
+export function watchInjectedEdits(sourceDir, targetDirs) {
+    const editedSinceMs = Date.now() - 2000;
+    let stopped = false;
+    let timer;
+    let inFlight;
+    const publish = () => {
+        if (stopped || inFlight != null)
+            return;
+        inFlight = publishToTargets(sourceDir, targetDirs, editedSinceMs).then(() => {
+            inFlight = undefined;
+        });
+    };
+    publish();
+    timer = setInterval(publish, 200);
+    return {
+        stop: async () => {
+            stopped = true;
+            if (timer != null)
+                clearInterval(timer);
+            await inFlight;
+        },
+    };
+}
+async function publishToTargets(sourceDir, targetDirs, editedSinceMs) {
+    let source;
+    try {
+        source = await readPublishSource(sourceDir);
+    }
+    catch (err) {
+        watchLogger.debug({
+            err,
+            message: `Failed to read injected dependency ${sourceDir} while its script is running`,
+        });
+        return;
+    }
+    await Promise.all(targetDirs.map(async (targetDir) => {
+        try {
+            await publishEditsForWatchers(source, targetDir, editedSinceMs);
+        }
+        catch (err) {
+            watchLogger.debug({
+                err,
+                message: `Failed to publish injected dependency ${targetDir} while its script is running`,
+            });
+        }
+    }));
+}
 export async function syncInjectedDeps(opts) {
     if (!opts.pkgName) {
         logger.debug({
@@ -22,10 +101,8 @@ export async function syncInjectedDeps(opts) {
     if (!opts.workspaceDir) {
         throw new PnpmError('NO_WORKSPACE_DIR', 'Cannot update injected dependencies without workspace dir');
     }
-    const pkgRootDir = path.resolve(opts.workspaceDir, opts.pkgRootDir);
-    const modulesDir = path.resolve(opts.workspaceDir, 'node_modules');
-    const modules = await readModulesManifest(modulesDir);
-    if (!modules?.injectedDeps) {
+    const located = await readInjectedTargets(opts.workspaceDir, opts.pkgRootDir);
+    if (located == null) {
         logger.debug({
             reason: 'no-injected-deps',
             message: 'Skipping sync of injected dependencies because none were detected',
@@ -33,9 +110,8 @@ export async function syncInjectedDeps(opts) {
         });
         return;
     }
-    const injectedDepKey = normalizePath(path.relative(opts.workspaceDir, pkgRootDir), true);
-    const targetDirs = modules.injectedDeps[injectedDepKey];
-    if (!targetDirs || targetDirs.length === 0) {
+    const { modules, pkgRootDir, resolvedTargetDirs } = located;
+    if (resolvedTargetDirs == null) {
         logger.debug({
             reason: 'no-injected-deps',
             message: `There are no injected dependencies from ${opts.pkgRootDir}`,
@@ -43,7 +119,6 @@ export async function syncInjectedDeps(opts) {
         });
         return;
     }
-    const resolvedTargetDirs = targetDirs.map(targetDir => path.resolve(opts.workspaceDir, targetDir));
     const patchers = await DirPatcher.fromMultipleTargets(pkgRootDir, resolvedTargetDirs);
     await Promise.all(patchers.map(patcher => patcher.apply()));
     await syncBinLinks({
@@ -59,6 +134,55 @@ export async function syncInjectedDeps(opts) {
         workspaceDir: opts.workspaceDir,
     });
 }
+/**
+ * Brings the injected copies listed in one modules directory back in step
+ * with their sources. A project with its own lockfile injects copies of
+ * workspace projects whose lifecycle scripts run in their own install, so
+ * nothing else syncs those copies after the scripts run.
+ */
+export async function syncInjectedDepsOfModulesDir(opts) {
+    const modules = await readModulesManifest(opts.modulesDir);
+    if (!modules?.injectedDeps)
+        return;
+    await Promise.all(Object.entries(modules.injectedDeps).map(async ([sourceId, targetDirs]) => {
+        const sourceDir = path.resolve(opts.lockfileDir, sourceId);
+        if (!opts.sourceDirs.has(sourceDir) || targetDirs.length === 0)
+            return;
+        // A publish directory the build did not produce leaves the copies alone
+        // rather than emptying them.
+        if (!await dirExists(sourceDir))
+            return;
+        const resolvedTargetDirs = targetDirs.map((targetDir) => path.resolve(opts.lockfileDir, targetDir));
+        const patchers = await DirPatcher.fromMultipleTargets(sourceDir, resolvedTargetDirs);
+        await Promise.all(patchers.map(patcher => patcher.apply()));
+    }));
+}
+async function dirExists(dir) {
+    try {
+        await fs.promises.stat(dir);
+        return true;
+    }
+    catch (err) {
+        if (isError(err) && 'code' in err && err.code === 'ENOENT')
+            return false;
+        throw err;
+    }
+}
+async function readInjectedTargets(workspaceDir, pkgRootDirInput) {
+    const pkgRootDir = path.resolve(workspaceDir, pkgRootDirInput);
+    const modules = await readModulesManifest(path.resolve(workspaceDir, 'node_modules'));
+    if (modules?.injectedDeps == null)
+        return undefined;
+    const injectedDepKey = normalizePath(path.relative(workspaceDir, pkgRootDir), true);
+    const targetDirs = modules.injectedDeps[injectedDepKey];
+    return {
+        modules,
+        pkgRootDir,
+        resolvedTargetDirs: targetDirs == null || targetDirs.length === 0
+            ? undefined
+            : targetDirs.map(targetDir => path.resolve(workspaceDir, targetDir)),
+    };
+}
 /** The commands a package declares, or none when it declares no bins. */
 async function readBinNames(pkgDir) {
     const manifest = await safeReadPackageJsonFromDir(pkgDir);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pnpm/workspace.injected-deps-syncer",
-  "version": "1100.0.38",
+  "version": "1100.0.40",
   "description": "Update all injected replica of a workspace package",
   "keywords": [
     "pnpm",
@@ -27,18 +27,20 @@
     "!*.map"
   ],
   "dependencies": {
-    "@pnpm/bins.linker": "1100.0.32",
-    "@pnpm/bins.remover": "1100.0.24",
-    "@pnpm/bins.resolver": "1100.0.16",
-    "@pnpm/error": "1100.1.4",
-    "@pnpm/fetching.directory-fetcher": "1100.0.34",
-    "@pnpm/installing.modules-yaml": "1101.0.3",
-    "@pnpm/pkg-manifest.reader": "1100.0.19",
+    "@pnpm/bins.linker": "1100.0.34",
+    "@pnpm/bins.remover": "1100.0.26",
+    "@pnpm/bins.resolver": "1100.0.17",
+    "@pnpm/error": "1100.2.1",
+    "@pnpm/fetching.directory-fetcher": "1100.0.36",
+    "@pnpm/fs.graceful-fs": "1100.2.4",
+    "@pnpm/installing.modules-yaml": "1101.0.5",
+    "@pnpm/pkg-manifest.reader": "1100.0.21",
     "@pnpm/text.ordinal-comparator": "1100.0.0",
-    "@pnpm/types": "1102.1.0",
-    "@pnpm/workspace.projects-reader": "1101.0.28",
+    "@pnpm/types": "1102.1.1",
+    "@pnpm/workspace.projects-reader": "1101.1.1",
     "@types/normalize-path": "^3.0.2",
-    "normalize-path": "^3.0.0"
+    "normalize-path": "^3.0.0",
+    "path-temp": "^3.0.0"
   },
   "peerDependencies": {
     "@pnpm/logger": "^1100.0.0"
@@ -46,8 +48,8 @@
   "devDependencies": {
     "@jest/globals": "30.4.1",
     "@pnpm/logger": "1100.0.0",
-    "@pnpm/prepare": "1100.0.31",
-    "@pnpm/workspace.injected-deps-syncer": "1100.0.38",
+    "@pnpm/prepare": "1100.0.33",
+    "@pnpm/workspace.injected-deps-syncer": "1100.0.40",
     "@types/is-windows": "^1.0.2",
     "is-windows": "^1.0.2"
   },