@pnpm/fs.indexed-pkg-importer 1100.0.19 → 1100.0.21

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/CHANGELOG.md CHANGED
@@ -1,5 +1,24 @@
1
1
  # @pnpm/fs.indexed-pkg-importer
2
2
 
3
+ ## 1100.0.21
4
+
5
+ ### Patch Changes
6
+
7
+ - Republished every package: the tarballs published by the v11.13.1 through v11.16.0 releases were missing most of their compiled files due to a packing bug [#13164](https://github.com/pnpm/pnpm/issues/13164).
8
+
9
+ - Updated dependencies:
10
+ - @pnpm/core-loggers@1100.2.5
11
+ - @pnpm/fs.graceful-fs@1100.1.1
12
+ - @pnpm/store.controller-types@1100.1.10
13
+
14
+ ## 1100.0.20
15
+
16
+ ### Patch Changes
17
+
18
+ - Updated dependencies:
19
+ - @pnpm/core-loggers@1100.2.4
20
+ - @pnpm/store.controller-types@1100.1.9
21
+
3
22
  ## 1100.0.19
4
23
 
5
24
  ### Patch Changes
@@ -0,0 +1,9 @@
1
+ export type ImportFile = (src: string, dest: string) => void;
2
+ export interface Importer {
3
+ importFile: ImportFile;
4
+ importFileAtomic: ImportFile;
5
+ }
6
+ export declare function importIndexedDir(importer: Importer, newDir: string, filenames: Map<string, string>, opts: {
7
+ keepModulesDir?: boolean;
8
+ safeToSkip?: boolean;
9
+ }): void;
@@ -0,0 +1,284 @@
1
+ import fs from 'node:fs';
2
+ import path from 'node:path';
3
+ import util from 'node:util';
4
+ import gfs from '@pnpm/fs.graceful-fs';
5
+ import { globalInfo, globalWarn, logger } from '@pnpm/logger';
6
+ import { rimrafSync } from '@zkochan/rimraf';
7
+ import fsx from 'fs-extra';
8
+ import { makeEmptyDirSync } from 'make-empty-dir';
9
+ import { fastPathTemp as pathTemp } from 'path-temp';
10
+ import { renameOverwriteSync } from 'rename-overwrite';
11
+ import sanitizeFilename from 'sanitize-filename';
12
+ const filenameConflictsLogger = logger('_filename-conflicts');
13
+ export function importIndexedDir(importer, newDir, filenames, opts) {
14
+ // Fast path: import directly without staging. Callers already verified
15
+ // the target package is missing (pkgExistsAtTargetDir / pkgLinkedToStore),
16
+ // so we can write straight into newDir and skip the temp dir + rename.
17
+ // On failure we fall through to the staging path, which has full error
18
+ // handling (EEXIST dedup, ENOENT sanitized-filename retry, etc.) and
19
+ // atomically swaps in a complete directory.
20
+ // keepModulesDir needs the staging path to preserve the existing node_modules.
21
+ if (!opts.keepModulesDir) {
22
+ if (opts.safeToSkip) {
23
+ // Content-addressed target (e.g. global virtual store): the path is
24
+ // shared across projects, so concurrent importers are expected. Use a
25
+ // non-destructive mkdir and let importFile dedup via EEXIST. If another
26
+ // importer already completed the directory, there's nothing to do.
27
+ try {
28
+ fs.mkdirSync(newDir, { recursive: true });
29
+ tryImportIndexedDir(importer, newDir, filenames);
30
+ return;
31
+ }
32
+ catch (err) {
33
+ if (util.types.isNativeError(err) && 'code' in err && err.code === 'EEXIST' && allFilesMatch(newDir, filenames)) {
34
+ return;
35
+ }
36
+ }
37
+ }
38
+ else if (tryExclusiveImport(importer, newDir, filenames)) {
39
+ return;
40
+ }
41
+ }
42
+ // Staging path: create in temp dir, then atomically rename.
43
+ // The dir rename is itself atomic, so individual file atomicity is not
44
+ // needed here — use importFile for everything.
45
+ const stage = pathTemp(newDir);
46
+ try {
47
+ makeEmptyDirSync(stage, { recursive: true });
48
+ tryImportIndexedDir({ importFile: importer.importFile, importFileAtomic: importer.importFile }, stage, filenames);
49
+ if (opts.keepModulesDir) {
50
+ // Keeping node_modules is needed only when the hoisted node linker is used.
51
+ moveOrMergeModulesDirs(path.join(newDir, 'node_modules'), path.join(stage, 'node_modules'));
52
+ }
53
+ }
54
+ catch (err) {
55
+ try {
56
+ rimrafSync(stage);
57
+ }
58
+ catch { } // eslint-disable-line:no-empty
59
+ if (util.types.isNativeError(err) && 'code' in err && err.code === 'EEXIST') {
60
+ const { uniqueFileMap, conflictingFileNames } = getUniqueFileMap(filenames);
61
+ if (conflictingFileNames.size === 0)
62
+ throw err;
63
+ filenameConflictsLogger.debug({
64
+ conflicts: Object.fromEntries(conflictingFileNames),
65
+ writingTo: newDir,
66
+ });
67
+ globalWarn(`Not all files were linked to "${path.relative(process.cwd(), newDir)}". ` +
68
+ 'Some of the files have equal names in different case, ' +
69
+ 'which is an issue on case-insensitive filesystems. ' +
70
+ `The conflicting file names are: ${JSON.stringify(Object.fromEntries(conflictingFileNames))}`);
71
+ importIndexedDir(importer, newDir, uniqueFileMap, opts);
72
+ return;
73
+ }
74
+ if (util.types.isNativeError(err) && 'code' in err && err.code === 'ENOENT') {
75
+ if (retryWithSanitizedFilenames(importer, newDir, filenames, opts))
76
+ return;
77
+ throw err;
78
+ }
79
+ throw err;
80
+ }
81
+ if (opts.safeToSkip) {
82
+ // Content-addressable target (e.g. global virtual store): if the target
83
+ // already exists and has all expected files, it has the correct content.
84
+ // Skip instead of doing a swap-rename that temporarily removes the target
85
+ // directory — which breaks junctions read by other processes.
86
+ try {
87
+ fs.renameSync(stage, newDir);
88
+ return;
89
+ }
90
+ catch (err) {
91
+ if (util.types.isNativeError(err) && 'code' in err && (err.code === 'ENOTEMPTY' || err.code === 'EEXIST' || err.code === 'EPERM')) {
92
+ if (allFilesMatch(newDir, filenames)) {
93
+ try {
94
+ rimrafSync(stage);
95
+ }
96
+ catch { } // eslint-disable-line:no-empty
97
+ return;
98
+ }
99
+ }
100
+ // Files missing or other error — fall through to renameOverwriteSync
101
+ }
102
+ }
103
+ try {
104
+ renameOverwriteSync(stage, newDir);
105
+ }
106
+ catch (renameErr) {
107
+ try {
108
+ rimrafSync(stage);
109
+ }
110
+ catch { } // eslint-disable-line:no-empty
111
+ throw renameErr;
112
+ }
113
+ }
114
+ // Fast path for the regular virtual store: write directly into newDir, but
115
+ // only when we can create it exclusively. A successful exclusive mkdir proves
116
+ // no other process is importing the same package concurrently, so the direct
117
+ // (non-atomic) write is safe. If newDir already exists — a concurrent importer
118
+ // or a stale partial directory from an interrupted import — this returns false
119
+ // so the caller falls back to the staging path, which builds a complete
120
+ // directory in a private temp dir and atomically renames it into place.
121
+ //
122
+ // Destructively emptying a directory another process may be populating could
123
+ // otherwise leave a partial package behind: if the surviving files include the
124
+ // package.json completion marker, every later install treats the broken
125
+ // directory as complete and never repairs it.
126
+ function tryExclusiveImport(importer, newDir, filenames) {
127
+ fs.mkdirSync(path.dirname(newDir), { recursive: true });
128
+ try {
129
+ fs.mkdirSync(newDir);
130
+ }
131
+ catch (err) {
132
+ if (util.types.isNativeError(err) && 'code' in err && err.code === 'EEXIST')
133
+ return false;
134
+ throw err;
135
+ }
136
+ // We exclusively created newDir, so no other process writes into it directly
137
+ // (concurrent importers see EEXIST above and take the staging path). The
138
+ // directory is ours: on failure we remove our partial result before falling
139
+ // back to staging, so the next attempt — including this process's own method
140
+ // fallbacks (clone → hardlink → copy) — can fast-path again.
141
+ try {
142
+ tryImportIndexedDir(importer, newDir, filenames);
143
+ return true;
144
+ }
145
+ catch {
146
+ try {
147
+ rimrafSync(newDir);
148
+ }
149
+ catch { } // eslint-disable-line:no-empty
150
+ return false;
151
+ }
152
+ }
153
+ function allFilesMatch(dir, filenames) {
154
+ for (const [f, src] of filenames) {
155
+ const target = path.join(dir, f);
156
+ try {
157
+ const targetStat = gfs.statSync(target);
158
+ const srcStat = gfs.statSync(src);
159
+ // Fast path: hardlinks share the same inode
160
+ if (targetStat.ino === srcStat.ino && targetStat.dev === srcStat.dev)
161
+ continue;
162
+ // Copy path: compare size first, then content
163
+ if (targetStat.size !== srcStat.size) {
164
+ globalInfo(`Re-importing "${dir}" because file "${f}" has a different size`);
165
+ return false;
166
+ }
167
+ if (!gfs.readFileSync(target).equals(gfs.readFileSync(src))) {
168
+ globalInfo(`Re-importing "${dir}" because file "${f}" has different content`);
169
+ return false;
170
+ }
171
+ }
172
+ catch {
173
+ globalInfo(`Re-importing "${dir}" because file "${f}" is missing or unreadable`);
174
+ return false;
175
+ }
176
+ }
177
+ return true;
178
+ }
179
+ function retryWithSanitizedFilenames(importer, newDir, filenames, opts) {
180
+ const { sanitizedFilenames, invalidFilenames } = sanitizeFilenames(filenames);
181
+ if (invalidFilenames.length === 0)
182
+ return false;
183
+ globalWarn(`\
184
+ The package linked to "${path.relative(process.cwd(), newDir)}" had \
185
+ files with invalid names: ${invalidFilenames.join(', ')}. \
186
+ They were renamed.`);
187
+ importIndexedDir(importer, newDir, sanitizedFilenames, opts);
188
+ return true;
189
+ }
190
+ function sanitizeFilenames(filenames) {
191
+ const sanitizedFilenames = new Map();
192
+ const invalidFilenames = [];
193
+ for (const [filename, src] of filenames) {
194
+ const sanitizedFilename = filename.split('/').map((f) => sanitizeFilename(f)).join('/');
195
+ if (sanitizedFilename !== filename) {
196
+ invalidFilenames.push(filename);
197
+ }
198
+ sanitizedFilenames.set(sanitizedFilename, src);
199
+ }
200
+ return { sanitizedFilenames, invalidFilenames };
201
+ }
202
+ function tryImportIndexedDir({ importFile, importFileAtomic }, newDir, filenames) {
203
+ const allDirs = new Set();
204
+ for (const f of filenames.keys()) {
205
+ const dir = path.dirname(f);
206
+ if (dir === '.')
207
+ continue;
208
+ allDirs.add(dir);
209
+ }
210
+ Array.from(allDirs)
211
+ .sort((d1, d2) => d1.length - d2.length) // from shortest to longest
212
+ .forEach((dir) => fs.mkdirSync(path.join(newDir, dir), { recursive: true }));
213
+ // Write package.json last so it acts as a completion marker.
214
+ // pkgExistsAtTargetDir() checks for package.json to decide if a package
215
+ // is already imported — writing it last ensures a crash mid-import won't
216
+ // leave a partially-populated directory that appears fully imported.
217
+ let packageJsonSrc;
218
+ for (const [f, src] of filenames) {
219
+ if (f === 'package.json') {
220
+ packageJsonSrc = src;
221
+ continue;
222
+ }
223
+ importFile(src, path.join(newDir, f));
224
+ }
225
+ if (packageJsonSrc !== undefined) {
226
+ importFileAtomic(packageJsonSrc, path.join(newDir, 'package.json'));
227
+ }
228
+ }
229
+ function getUniqueFileMap(fileMap) {
230
+ const lowercaseFiles = new Map();
231
+ const conflictingFileNames = new Map();
232
+ const uniqueFileMap = new Map();
233
+ for (const filename of Array.from(fileMap.keys()).sort()) {
234
+ const lowercaseFilename = filename.toLowerCase();
235
+ if (lowercaseFiles.has(lowercaseFilename)) {
236
+ conflictingFileNames.set(filename, lowercaseFiles.get(lowercaseFilename));
237
+ continue;
238
+ }
239
+ lowercaseFiles.set(lowercaseFilename, filename);
240
+ uniqueFileMap.set(filename, fileMap.get(filename));
241
+ }
242
+ return {
243
+ conflictingFileNames,
244
+ uniqueFileMap,
245
+ };
246
+ }
247
+ function moveOrMergeModulesDirs(src, dest) {
248
+ try {
249
+ renameEvenAcrossDevices(src, dest);
250
+ }
251
+ catch (err) {
252
+ switch (util.types.isNativeError(err) && 'code' in err && err.code) {
253
+ case 'ENOENT':
254
+ // If src directory doesn't exist, there is nothing to do
255
+ return;
256
+ case 'ENOTEMPTY':
257
+ case 'EPERM': // This error code is thrown on Windows
258
+ // The newly added dependency might have node_modules if it has bundled dependencies.
259
+ mergeModulesDirs(src, dest);
260
+ return;
261
+ default:
262
+ throw err;
263
+ }
264
+ }
265
+ }
266
+ function renameEvenAcrossDevices(src, dest) {
267
+ try {
268
+ gfs.renameSync(src, dest);
269
+ }
270
+ catch (err) {
271
+ if (!(util.types.isNativeError(err) && 'code' in err && err.code === 'EXDEV'))
272
+ throw err;
273
+ fsx.copySync(src, dest);
274
+ }
275
+ }
276
+ function mergeModulesDirs(src, dest) {
277
+ const srcFiles = fs.readdirSync(src);
278
+ const destFiles = new Set(fs.readdirSync(dest));
279
+ const filesToMove = srcFiles.filter((file) => !destFiles.has(file));
280
+ for (const file of filesToMove) {
281
+ renameEvenAcrossDevices(path.join(src, file), path.join(dest, file));
282
+ }
283
+ }
284
+ //# sourceMappingURL=importIndexedDir.js.map
package/lib/index.d.ts ADDED
@@ -0,0 +1,5 @@
1
+ import type { FilesMap, ImportIndexedPackage, ImportOptions } from '@pnpm/store.controller-types';
2
+ export { type FilesMap, type ImportIndexedPackage, type ImportOptions };
3
+ export type PackageImportMethod = 'auto' | 'hardlink' | 'copy' | 'clone' | 'clone-or-copy';
4
+ export declare function createIndexedPkgImporter(packageImportMethod?: PackageImportMethod): ImportIndexedPackage;
5
+ export declare function copyPkg(to: string, opts: ImportOptions): 'copy' | undefined;
@@ -0,0 +1,23 @@
1
+ /**
2
+ * Native binaries are the only files that macOS Gatekeeper blocks for carrying a
3
+ * quarantine xattr, so removing it from anything else (JavaScript, text, etc.)
4
+ * just wastes a syscall.
5
+ */
6
+ export declare function isNativeBinary(filePath: string): boolean;
7
+ /**
8
+ * Remove the macOS Gatekeeper quarantine xattr (com.apple.quarantine) from the
9
+ * given files using a single `xattr` invocation.
10
+ *
11
+ * macOS preserves extended attributes when pnpm copies or reflinks files out of
12
+ * its content-addressable store. If a store blob carries com.apple.quarantine
13
+ * (e.g. it was first written under a Gatekeeper-enabled app such as a Git
14
+ * client), the quarantine propagates to node_modules and Gatekeeper blocks the
15
+ * native binary from loading, even though pnpm already verified the file's
16
+ * integrity against the lockfile. This mirrors Homebrew's behaviour of dropping
17
+ * quarantine from verified downloads.
18
+ *
19
+ * File paths are passed as separate arguments rather than interpolated into a
20
+ * shell command, so package-controlled filenames cannot inject shell commands.
21
+ * They are split into chunks that stay under the OS argv limit.
22
+ */
23
+ export declare function removeQuarantine(filePaths: string[]): void;
@@ -0,0 +1,90 @@
1
+ import { execFileSync } from 'node:child_process';
2
+ import path from 'node:path';
3
+ import { globalWarn } from '@pnpm/logger';
4
+ const QUARANTINE_ATTR = 'com.apple.quarantine';
5
+ // Quarantine removal only ever runs on macOS, so the set lists the binary
6
+ // formats Gatekeeper guards there (.dll is Windows-only and never matches).
7
+ const NATIVE_BINARY_EXTENSIONS = new Set(['.node', '.dylib', '.so']);
8
+ // Cap the bytes of file-path arguments per `xattr` call so a package with many
9
+ // (or very long) native-binary paths can't blow past the OS argv limit
10
+ // (ARG_MAX, ~1 MB on macOS). Well under the limit to leave room for argv0 and
11
+ // the environment block.
12
+ const MAX_ARG_BYTES = 100_000;
13
+ /**
14
+ * Native binaries are the only files that macOS Gatekeeper blocks for carrying a
15
+ * quarantine xattr, so removing it from anything else (JavaScript, text, etc.)
16
+ * just wastes a syscall.
17
+ */
18
+ export function isNativeBinary(filePath) {
19
+ return NATIVE_BINARY_EXTENSIONS.has(path.extname(filePath).toLowerCase());
20
+ }
21
+ /**
22
+ * Remove the macOS Gatekeeper quarantine xattr (com.apple.quarantine) from the
23
+ * given files using a single `xattr` invocation.
24
+ *
25
+ * macOS preserves extended attributes when pnpm copies or reflinks files out of
26
+ * its content-addressable store. If a store blob carries com.apple.quarantine
27
+ * (e.g. it was first written under a Gatekeeper-enabled app such as a Git
28
+ * client), the quarantine propagates to node_modules and Gatekeeper blocks the
29
+ * native binary from loading, even though pnpm already verified the file's
30
+ * integrity against the lockfile. This mirrors Homebrew's behaviour of dropping
31
+ * quarantine from verified downloads.
32
+ *
33
+ * File paths are passed as separate arguments rather than interpolated into a
34
+ * shell command, so package-controlled filenames cannot inject shell commands.
35
+ * They are split into chunks that stay under the OS argv limit.
36
+ */
37
+ export function removeQuarantine(filePaths) {
38
+ if (process.platform !== 'darwin')
39
+ return;
40
+ for (const chunk of chunkByArgSize(filePaths)) {
41
+ removeQuarantineFromChunk(chunk);
42
+ }
43
+ }
44
+ function removeQuarantineFromChunk(filePaths) {
45
+ try {
46
+ execFileSync('/usr/bin/xattr', ['-d', QUARANTINE_ATTR, ...filePaths], {
47
+ stdio: ['ignore', 'ignore', 'pipe'],
48
+ });
49
+ }
50
+ catch (err) {
51
+ // `xattr -d` exits non-zero when a file simply has no quarantine xattr to
52
+ // remove ("No such xattr"), which is the common, expected case. It also
53
+ // reports "No such file" for entries the importer legitimately dropped or
54
+ // renamed (e.g. case-insensitive filename conflicts). Surface only errors
55
+ // that are not of those kinds (e.g. permission denied).
56
+ const realErrors = getStderr(err)
57
+ .split('\n')
58
+ .filter((line) => line.trim() !== '' && !line.includes('No such xattr') && !line.includes('No such file'));
59
+ if (realErrors.length > 0) {
60
+ globalWarn(`Failed to remove the macOS quarantine attribute:\n${realErrors.join('\n')}`);
61
+ }
62
+ }
63
+ }
64
+ function chunkByArgSize(filePaths) {
65
+ const chunks = [];
66
+ let chunk = [];
67
+ let chunkBytes = 0;
68
+ for (const filePath of filePaths) {
69
+ const bytes = Buffer.byteLength(filePath) + 1; // +1 for the argv null terminator
70
+ if (chunk.length > 0 && chunkBytes + bytes > MAX_ARG_BYTES) {
71
+ chunks.push(chunk);
72
+ chunk = [];
73
+ chunkBytes = 0;
74
+ }
75
+ chunk.push(filePath);
76
+ chunkBytes += bytes;
77
+ }
78
+ if (chunk.length > 0)
79
+ chunks.push(chunk);
80
+ return chunks;
81
+ }
82
+ function getStderr(err) {
83
+ if (typeof err === 'object' && err !== null && 'stderr' in err) {
84
+ const stderr = err.stderr;
85
+ if (stderr != null)
86
+ return stderr.toString();
87
+ }
88
+ return err instanceof Error ? err.message : String(err);
89
+ }
90
+ //# sourceMappingURL=removeQuarantine.js.map
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@pnpm/fs.indexed-pkg-importer",
3
- "version": "1100.0.19",
3
+ "version": "1100.0.21",
4
4
  "description": "Replicates indexed directories using hard links, copies, or cloning",
5
5
  "keywords": [
6
6
  "pnpm",
@@ -37,9 +37,9 @@
37
37
  "test": "test"
38
38
  },
39
39
  "dependencies": {
40
- "@pnpm/core-loggers": "1100.2.3",
41
- "@pnpm/fs.graceful-fs": "1100.1.0",
42
- "@pnpm/store.controller-types": "1100.1.8",
40
+ "@pnpm/core-loggers": "1100.2.5",
41
+ "@pnpm/fs.graceful-fs": "1100.1.1",
42
+ "@pnpm/store.controller-types": "1100.1.10",
43
43
  "@reflink/reflink": "0.1.19",
44
44
  "@zkochan/rimraf": "^4.0.0",
45
45
  "fs-extra": "^11.3.5",
@@ -54,9 +54,9 @@
54
54
  },
55
55
  "devDependencies": {
56
56
  "@jest/globals": "30.4.1",
57
- "@pnpm/fs.indexed-pkg-importer": "1100.0.19",
57
+ "@pnpm/fs.indexed-pkg-importer": "1100.0.21",
58
58
  "@pnpm/logger": "1100.0.0",
59
- "@pnpm/prepare": "1100.0.20",
59
+ "@pnpm/prepare": "1100.0.22",
60
60
  "@types/fs-extra": "^11.0.4"
61
61
  },
62
62
  "engines": {