@pnpm/fs.indexed-pkg-importer 1100.0.23 → 1100.0.25

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/CHANGELOG.md CHANGED
@@ -1,5 +1,22 @@
1
1
  # @pnpm/fs.indexed-pkg-importer
2
2
 
3
+ ## 1100.0.25
4
+
5
+ ### Patch Changes
6
+
7
+ - An install sharing a global virtual store no longer removes an incomplete package directory that another importer is still writing, which could fail with `failed to remove existing directory ... prior to swap: Directory not empty`. Such a directory is now repaired in place, and a package file left damaged by an interrupted install is restored instead of being kept.
8
+
9
+ - Updated dependencies:
10
+ - @pnpm/store.controller-types@1101.1.1
11
+
12
+ ## 1100.0.24
13
+
14
+ ### Patch Changes
15
+
16
+ - Updated dependencies:
17
+ - @pnpm/core-loggers@1100.3.2
18
+ - @pnpm/store.controller-types@1101.1.0
19
+
3
20
  ## 1100.0.23
4
21
 
5
22
  ### Patch Changes
@@ -1,9 +1,16 @@
1
+ import type { ResolvedFrom } from '@pnpm/store.controller-types';
1
2
  export type ImportFile = (src: string, dest: string) => void;
2
3
  export interface Importer {
3
4
  importFile: ImportFile;
4
5
  importFileAtomic: ImportFile;
5
6
  }
6
- export declare function importIndexedDir(importer: Importer, newDir: string, filenames: Map<string, string>, opts: {
7
+ export interface ImportIndexedDirOptions {
7
8
  keepModulesDir?: boolean;
9
+ /**
10
+ * Whether a target that already holds this package is equivalent to the
11
+ * import, which requires that the target path pin its contents.
12
+ */
8
13
  safeToSkip?: boolean;
9
- }): void;
14
+ resolvedFrom?: ResolvedFrom;
15
+ }
16
+ export declare function importIndexedDir(importer: Importer, newDir: string, filenames: Map<string, string>, opts: ImportIndexedDirOptions): void;
@@ -10,7 +10,26 @@ import { fastPathTemp as pathTemp } from 'path-temp';
10
10
  import { renameOverwriteSync } from 'rename-overwrite';
11
11
  import sanitizeFilename from 'sanitize-filename';
12
12
  const filenameConflictsLogger = logger('_filename-conflicts');
13
+ const RENAME_RETRY_BUDGET_MS = 60_000;
14
+ const RENAME_RETRY_BACKOFF_CAP_MS = 100;
15
+ const FILE_COMPARE_BUFFER_SIZE = 64 * 1024;
16
+ const renameRetrySleepBuffer = new Int32Array(new SharedArrayBuffer(4));
13
17
  export function importIndexedDir(importer, newDir, filenames, opts) {
18
+ const dirImport = { importer, newDir, filenames, opts };
19
+ // Content-addressed target (e.g. global virtual store): the path is shared
20
+ // across projects, so concurrent importers are expected and the directory
21
+ // must never be removed or swapped out from under them. It is populated in
22
+ // place, adopted when it already matches, and otherwise repaired entry by
23
+ // entry. Repairing in place leaves a nested node_modules/ where it is, so
24
+ // keepModulesDir has nothing to preserve here either.
25
+ //
26
+ // A local directory is copied into the target at install time, so its
27
+ // contents can change without the lockfile changing and the path pins
28
+ // nothing. Such a target is rebuilt below rather than adopted or repaired.
29
+ if (opts.safeToSkip && opts.resolvedFrom !== 'local-dir') {
30
+ importIntoSharedDir(dirImport);
31
+ return;
32
+ }
14
33
  // Fast path: import directly without staging. Callers already verified
15
34
  // the target package is missing (pkgExistsAtTargetDir / pkgLinkedToStore),
16
35
  // so we can write straight into newDir and skip the temp dir + rename.
@@ -18,26 +37,8 @@ export function importIndexedDir(importer, newDir, filenames, opts) {
18
37
  // handling (EEXIST dedup, ENOENT sanitized-filename retry, etc.) and
19
38
  // atomically swaps in a complete directory.
20
39
  // keepModulesDir needs the staging path to preserve the existing node_modules.
21
- if (!opts.keepModulesDir) {
22
- if (opts.safeToSkip) {
23
- // Content-addressed target (e.g. global virtual store): the path is
24
- // shared across projects, so concurrent importers are expected. Use a
25
- // non-destructive mkdir and let importFile dedup via EEXIST. If another
26
- // importer already completed the directory, there's nothing to do.
27
- try {
28
- fs.mkdirSync(newDir, { recursive: true });
29
- tryImportIndexedDir(importer, newDir, filenames);
30
- return;
31
- }
32
- catch (err) {
33
- if (util.types.isNativeError(err) && 'code' in err && err.code === 'EEXIST' && allFilesMatch(newDir, filenames)) {
34
- return;
35
- }
36
- }
37
- }
38
- else if (tryExclusiveImport(importer, newDir, filenames)) {
39
- return;
40
- }
40
+ if (!opts.keepModulesDir && tryExclusiveImport(importer, newDir, filenames)) {
41
+ return;
41
42
  }
42
43
  // Staging path: create in temp dir, then atomically rename.
43
44
  // The dir rename is itself atomic, so individual file atomicity is not
@@ -56,60 +57,209 @@ export function importIndexedDir(importer, newDir, filenames, opts) {
56
57
  rimrafSync(stage);
57
58
  }
58
59
  catch { } // eslint-disable-line:no-empty
59
- if (util.types.isNativeError(err) && 'code' in err && err.code === 'EEXIST') {
60
- const { uniqueFileMap, conflictingFileNames } = getUniqueFileMap(filenames);
61
- if (conflictingFileNames.size === 0)
62
- throw err;
63
- filenameConflictsLogger.debug({
64
- conflicts: Object.fromEntries(conflictingFileNames),
65
- writingTo: newDir,
66
- });
67
- globalWarn(`Not all files were linked to "${path.relative(process.cwd(), newDir)}". ` +
68
- 'Some of the files have equal names in different case, ' +
69
- 'which is an issue on case-insensitive filesystems. ' +
70
- `The conflicting file names are: ${JSON.stringify(Object.fromEntries(conflictingFileNames))}`);
71
- importIndexedDir(importer, newDir, uniqueFileMap, opts);
60
+ if (retryWithFixedFileMap(err, dirImport))
72
61
  return;
62
+ throw err;
63
+ }
64
+ try {
65
+ renameOverwriteSync(stage, newDir);
66
+ }
67
+ catch (renameErr) {
68
+ try {
69
+ rimrafSync(stage);
73
70
  }
74
- if (util.types.isNativeError(err) && 'code' in err && err.code === 'ENOENT') {
75
- if (retryWithSanitizedFilenames(importer, newDir, filenames, opts))
76
- return;
71
+ catch { } // eslint-disable-line:no-empty
72
+ throw renameErr;
73
+ }
74
+ }
75
+ // Import into a directory whose path is shared with other projects, and with
76
+ // the installs running in them. Nothing here removes a dirent: the winner of
77
+ // the exclusive mkdir writes the package, and everyone else adopts the result
78
+ // when it already matches and otherwise replaces only the entries that do not,
79
+ // so concurrent importers converge on the same tree.
80
+ //
81
+ // Replacing rather than adopting is what keeps a shared slot repairable. The
82
+ // linking tiers report EEXIST for a dirent that is already there, so an import
83
+ // that adopts what it finds keeps a file truncated by an interrupted copy, and
84
+ // then puts the completion marker on top of it — after which the directory
85
+ // looks finished to every later install and is never repaired.
86
+ function importIntoSharedDir(dirImport) {
87
+ const { importer, newDir, filenames } = dirImport;
88
+ fs.mkdirSync(path.dirname(newDir), { recursive: true });
89
+ let created = false;
90
+ try {
91
+ fs.mkdirSync(newDir);
92
+ created = true;
93
+ }
94
+ catch (err) {
95
+ if (!util.types.isNativeError(err) || !('code' in err) || err.code !== 'EEXIST')
77
96
  throw err;
97
+ }
98
+ if (created) {
99
+ try {
100
+ tryImportIndexedDir(importer, newDir, filenames);
101
+ return;
102
+ }
103
+ catch (err) {
104
+ if (retryWithFixedFileMap(err, dirImport))
105
+ return;
106
+ // Our own write stopped partway. Another importer may already be reading
107
+ // what did land, so finish the directory in place instead of staging a
108
+ // replacement for it.
109
+ }
110
+ }
111
+ if (allFilesMatch(newDir, filenames))
112
+ return;
113
+ try {
114
+ repairIndexedDir(dirImport);
115
+ }
116
+ catch (err) {
117
+ if (retryWithFixedFileMap(err, dirImport))
118
+ return;
119
+ throw err;
120
+ }
121
+ }
122
+ // Bring an occupied shared directory up to date with the file map, entry by
123
+ // entry. Files the package does not declare are left alone: a build output
124
+ // belongs to whoever put it there, and a slot other installs are reading is
125
+ // not somewhere to delete from speculatively.
126
+ function repairIndexedDir({ importer, newDir, filenames }) {
127
+ makeFileMapDirs(newDir, filenames, { clearBlockers: true });
128
+ let packageJsonSrc;
129
+ for (const [f, src] of filenames) {
130
+ if (f === 'package.json') {
131
+ packageJsonSrc = src;
132
+ continue;
133
+ }
134
+ replaceFileIfDifferent(importer.importFile, src, path.join(newDir, f));
135
+ }
136
+ if (packageJsonSrc !== undefined) {
137
+ replaceFileIfDifferent(importer.importFile, packageJsonSrc, path.join(newDir, 'package.json'));
138
+ }
139
+ }
140
+ // Swap the file in through a temp sibling: a reader sees either the old dirent
141
+ // or the new one, and the rename replaces what the linking tiers would have
142
+ // refused to overwrite.
143
+ function replaceFileIfDifferent(importFile, src, dest) {
144
+ if (mismatchReason(dest, src) === undefined)
145
+ return;
146
+ const tmp = pathTemp(dest);
147
+ try {
148
+ importFile(src, tmp);
149
+ }
150
+ catch (err) {
151
+ try {
152
+ fs.unlinkSync(tmp);
153
+ }
154
+ catch { } // eslint-disable-line:no-empty
155
+ throw err;
156
+ }
157
+ try {
158
+ clearDirBlockingFile(dest);
159
+ renameFileWithRetry(tmp, dest);
160
+ }
161
+ catch (err) {
162
+ try {
163
+ fs.unlinkSync(tmp);
78
164
  }
165
+ catch { } // eslint-disable-line:no-empty
166
+ if (mismatchReason(dest, src) === undefined)
167
+ return;
79
168
  throw err;
80
169
  }
81
- if (opts.safeToSkip) {
82
- // Content-addressable target (e.g. global virtual store): if the target
83
- // already exists and has all expected files, it has the correct content.
84
- // Skip instead of doing a swap-rename that temporarily removes the target
85
- // directory — which breaks junctions read by other processes.
170
+ }
171
+ // Retry a Windows sharing violation without rename-overwrite's fallback of
172
+ // deleting the destination. Another install may still be reading that dirent.
173
+ function renameFileWithRetry(src, dest) {
174
+ const startedAt = Date.now();
175
+ let backoffMs = 0;
176
+ for (;;) {
86
177
  try {
87
- fs.renameSync(stage, newDir);
178
+ fs.renameSync(src, dest);
88
179
  return;
89
180
  }
90
181
  catch (err) {
91
- if (util.types.isNativeError(err) && 'code' in err && (err.code === 'ENOTEMPTY' || err.code === 'EEXIST' || err.code === 'EPERM')) {
92
- if (allFilesMatch(newDir, filenames)) {
93
- try {
94
- rimrafSync(stage);
95
- }
96
- catch { } // eslint-disable-line:no-empty
97
- return;
98
- }
99
- }
100
- // Files missing or other error — fall through to renameOverwriteSync
182
+ if (!isTransientRenameError(err) || Date.now() - startedAt >= RENAME_RETRY_BUDGET_MS)
183
+ throw err;
184
+ if (backoffMs > 0)
185
+ Atomics.wait(renameRetrySleepBuffer, 0, 0, backoffMs);
186
+ backoffMs = Math.min(backoffMs + 10, RENAME_RETRY_BACKOFF_CAP_MS);
101
187
  }
102
188
  }
189
+ }
190
+ function isTransientRenameError(err) {
191
+ return process.platform === 'win32' &&
192
+ util.types.isNativeError(err) &&
193
+ 'code' in err &&
194
+ (err.code === 'EPERM' || err.code === 'EACCES' || err.code === 'EBUSY');
195
+ }
196
+ // A rename cannot put a file where a directory is (EISDIR), so one standing in
197
+ // the way has to go first. Only a damaged tree has one.
198
+ function clearDirBlockingFile(dest) {
199
+ let stats;
103
200
  try {
104
- renameOverwriteSync(stage, newDir);
201
+ stats = fs.lstatSync(dest);
105
202
  }
106
- catch (renameErr) {
203
+ catch (err) {
204
+ if (util.types.isNativeError(err) && 'code' in err && err.code === 'ENOENT')
205
+ return;
206
+ throw err;
207
+ }
208
+ if (stats.isDirectory()) {
209
+ rimrafSync(dest);
210
+ }
211
+ }
212
+ // A file where the package needs a directory turns up only in a damaged tree,
213
+ // and would fail the mkdir below. Walking top-down means a segment whose
214
+ // parent is itself a file is never reached: the parent goes first, and
215
+ // everything under a missing segment is missing too.
216
+ function clearDirentBlockingDir(newDir, relativeDir) {
217
+ let dir = newDir;
218
+ for (const segment of relativeDir.split(/[\\/]/)) {
219
+ dir = path.join(dir, segment);
220
+ let stats;
107
221
  try {
108
- rimrafSync(stage);
222
+ stats = fs.lstatSync(dir);
109
223
  }
110
- catch { } // eslint-disable-line:no-empty
111
- throw renameErr;
224
+ catch (err) {
225
+ if (util.types.isNativeError(err) && 'code' in err && err.code === 'ENOENT')
226
+ return;
227
+ throw err;
228
+ }
229
+ if (stats.isDirectory())
230
+ continue;
231
+ fs.unlinkSync(dir);
232
+ return;
233
+ }
234
+ }
235
+ // The two failures an indexed file map can cause on its own: names that
236
+ // collide on a case-insensitive filesystem, and names the filesystem rejects
237
+ // outright. Both are recovered by rewriting the map and importing again.
238
+ // Returns false for anything else, which the caller must treat as a real
239
+ // failure.
240
+ function retryWithFixedFileMap(err, dirImport) {
241
+ const { importer, newDir, filenames, opts } = dirImport;
242
+ if (!util.types.isNativeError(err) || !('code' in err))
243
+ return false;
244
+ if (err.code === 'EEXIST') {
245
+ const { uniqueFileMap, conflictingFileNames } = getUniqueFileMap(filenames);
246
+ if (conflictingFileNames.size === 0)
247
+ return false;
248
+ filenameConflictsLogger.debug({
249
+ conflicts: Object.fromEntries(conflictingFileNames),
250
+ writingTo: newDir,
251
+ });
252
+ globalWarn(`Not all files were linked to "${path.relative(process.cwd(), newDir)}". ` +
253
+ 'Some of the files have equal names in different case, ' +
254
+ 'which is an issue on case-insensitive filesystems. ' +
255
+ `The conflicting file names are: ${JSON.stringify(Object.fromEntries(conflictingFileNames))}`);
256
+ importIndexedDir(importer, newDir, uniqueFileMap, opts);
257
+ return true;
112
258
  }
259
+ if (err.code === 'ENOENT') {
260
+ return retryWithSanitizedFilenames(dirImport);
261
+ }
262
+ return false;
113
263
  }
114
264
  // Fast path for the regular virtual store: write directly into newDir, but
115
265
  // only when we can create it exclusively. A successful exclusive mkdir proves
@@ -151,32 +301,79 @@ function tryExclusiveImport(importer, newDir, filenames) {
151
301
  }
152
302
  }
153
303
  function allFilesMatch(dir, filenames) {
304
+ // The completion marker is written last, so its absence settles the common
305
+ // case — a directory another importer is still filling — in one stat,
306
+ // wherever the map happens to hold it.
307
+ const markerSrc = filenames.get('package.json');
308
+ if (markerSrc !== undefined && !fileMatches(dir, 'package.json', markerSrc))
309
+ return false;
154
310
  for (const [f, src] of filenames) {
155
- const target = path.join(dir, f);
311
+ if (f === 'package.json')
312
+ continue;
313
+ if (!fileMatches(dir, f, src))
314
+ return false;
315
+ }
316
+ return true;
317
+ }
318
+ function fileMatches(dir, f, src) {
319
+ const reason = mismatchReason(path.join(dir, f), src);
320
+ if (reason === undefined)
321
+ return true;
322
+ globalInfo(`Re-importing "${dir}" because file "${f}" ${reason}`);
323
+ return false;
324
+ }
325
+ // Why `target` is not the store file at `src`, or undefined when it already
326
+ // is. Files imported by hardlink or reflink share the store inode, which
327
+ // settles it without a read; the copy tier compares size first, then content.
328
+ function mismatchReason(target, src) {
329
+ try {
330
+ const targetStat = fs.lstatSync(target, { bigint: true });
331
+ if (!targetStat.isFile())
332
+ return 'is not a regular file';
333
+ const srcStat = gfs.statSync(src, { bigint: true });
334
+ const hasSameFileIdentity = targetStat.ino !== 0n &&
335
+ targetStat.dev !== 0n &&
336
+ targetStat.ino === srcStat.ino &&
337
+ targetStat.dev === srcStat.dev;
338
+ if (hasSameFileIdentity)
339
+ return undefined;
340
+ if (targetStat.size !== srcStat.size)
341
+ return 'has a different size';
342
+ if (!filesHaveEqualContents(target, src))
343
+ return 'has different content';
344
+ return undefined;
345
+ }
346
+ catch {
347
+ return 'is missing or unreadable';
348
+ }
349
+ }
350
+ function filesHaveEqualContents(left, right) {
351
+ const leftBuffer = Buffer.allocUnsafe(FILE_COMPARE_BUFFER_SIZE);
352
+ const rightBuffer = Buffer.allocUnsafe(FILE_COMPARE_BUFFER_SIZE);
353
+ const leftFd = fs.openSync(left, 'r');
354
+ try {
355
+ const rightFd = fs.openSync(right, 'r');
156
356
  try {
157
- const targetStat = gfs.statSync(target);
158
- const srcStat = gfs.statSync(src);
159
- // Fast path: hardlinks share the same inode
160
- if (targetStat.ino === srcStat.ino && targetStat.dev === srcStat.dev)
161
- continue;
162
- // Copy path: compare size first, then content
163
- if (targetStat.size !== srcStat.size) {
164
- globalInfo(`Re-importing "${dir}" because file "${f}" has a different size`);
165
- return false;
166
- }
167
- if (!gfs.readFileSync(target).equals(gfs.readFileSync(src))) {
168
- globalInfo(`Re-importing "${dir}" because file "${f}" has different content`);
169
- return false;
357
+ for (;;) {
358
+ const leftBytes = fs.readSync(leftFd, leftBuffer, 0, leftBuffer.length, null);
359
+ const rightBytes = fs.readSync(rightFd, rightBuffer, 0, rightBuffer.length, null);
360
+ if (leftBytes !== rightBytes)
361
+ return false;
362
+ if (leftBytes === 0)
363
+ return true;
364
+ if (!leftBuffer.subarray(0, leftBytes).equals(rightBuffer.subarray(0, rightBytes)))
365
+ return false;
170
366
  }
171
367
  }
172
- catch {
173
- globalInfo(`Re-importing "${dir}" because file "${f}" is missing or unreadable`);
174
- return false;
368
+ finally {
369
+ fs.closeSync(rightFd);
175
370
  }
176
371
  }
177
- return true;
372
+ finally {
373
+ fs.closeSync(leftFd);
374
+ }
178
375
  }
179
- function retryWithSanitizedFilenames(importer, newDir, filenames, opts) {
376
+ function retryWithSanitizedFilenames({ importer, newDir, filenames, opts }) {
180
377
  const { sanitizedFilenames, invalidFilenames } = sanitizeFilenames(filenames);
181
378
  if (invalidFilenames.length === 0)
182
379
  return false;
@@ -200,16 +397,7 @@ function sanitizeFilenames(filenames) {
200
397
  return { sanitizedFilenames, invalidFilenames };
201
398
  }
202
399
  function tryImportIndexedDir({ importFile, importFileAtomic }, newDir, filenames) {
203
- const allDirs = new Set();
204
- for (const f of filenames.keys()) {
205
- const dir = path.dirname(f);
206
- if (dir === '.')
207
- continue;
208
- allDirs.add(dir);
209
- }
210
- Array.from(allDirs)
211
- .sort((d1, d2) => d1.length - d2.length) // from shortest to longest
212
- .forEach((dir) => fs.mkdirSync(path.join(newDir, dir), { recursive: true }));
400
+ makeFileMapDirs(newDir, filenames);
213
401
  // Write package.json last so it acts as a completion marker.
214
402
  // pkgExistsAtTargetDir() checks for package.json to decide if a package
215
403
  // is already imported — writing it last ensures a crash mid-import won't
@@ -226,6 +414,23 @@ function tryImportIndexedDir({ importFile, importFileAtomic }, newDir, filenames
226
414
  importFileAtomic(packageJsonSrc, path.join(newDir, 'package.json'));
227
415
  }
228
416
  }
417
+ // Sorting shortest-first means the recursive mkdir for a deeper directory
418
+ // always finds its ancestor already on disk.
419
+ function makeFileMapDirs(newDir, filenames, opts) {
420
+ const allDirs = new Set();
421
+ for (const f of filenames.keys()) {
422
+ const dir = path.dirname(f);
423
+ if (dir === '.')
424
+ continue;
425
+ allDirs.add(dir);
426
+ }
427
+ for (const dir of Array.from(allDirs).sort((d1, d2) => d1.length - d2.length)) {
428
+ if (opts?.clearBlockers) {
429
+ clearDirentBlockingDir(newDir, dir);
430
+ }
431
+ fs.mkdirSync(path.join(newDir, dir), { recursive: true });
432
+ }
433
+ }
229
434
  function getUniqueFileMap(fileMap) {
230
435
  const lowercaseFiles = new Map();
231
436
  const conflictingFileNames = new Map();
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@pnpm/fs.indexed-pkg-importer",
3
- "version": "1100.0.23",
3
+ "version": "1100.0.25",
4
4
  "description": "Replicates indexed directories using hard links, copies, or cloning",
5
5
  "keywords": [
6
6
  "pnpm",
@@ -37,9 +37,9 @@
37
37
  "test": "test"
38
38
  },
39
39
  "dependencies": {
40
- "@pnpm/core-loggers": "1100.3.1",
40
+ "@pnpm/core-loggers": "1100.3.2",
41
41
  "@pnpm/fs.graceful-fs": "1100.1.1",
42
- "@pnpm/store.controller-types": "1101.0.0",
42
+ "@pnpm/store.controller-types": "1101.1.1",
43
43
  "@reflink/reflink": "0.1.19",
44
44
  "@zkochan/rimraf": "^4.0.0",
45
45
  "fs-extra": "^11.4.0",
@@ -54,9 +54,9 @@
54
54
  },
55
55
  "devDependencies": {
56
56
  "@jest/globals": "30.4.1",
57
- "@pnpm/fs.indexed-pkg-importer": "1100.0.23",
57
+ "@pnpm/fs.indexed-pkg-importer": "1100.0.25",
58
58
  "@pnpm/logger": "1100.0.0",
59
- "@pnpm/prepare": "1100.0.24",
59
+ "@pnpm/prepare": "1100.0.26",
60
60
  "@types/fs-extra": "^11.0.4"
61
61
  },
62
62
  "engines": {