@depup/webpack 5.106.1-depup.0 → 5.109.2-depup.0

package/lib/cache/MemoryWithGcCachePlugin.js

@@ -12,6 +12,7 @@ const Cache = require("../Cache");
 /** @typedef {import("../Compiler")} Compiler */
 
 /**
+ * Defines the memory with gc cache plugin options type used by this module.
  * @typedef {object} MemoryWithGcCachePluginOptions
  * @property {number} maxGenerations max generations
  */
@@ -20,14 +21,16 @@ const PLUGIN_NAME = "MemoryWithGcCachePlugin";
 
 class MemoryWithGcCachePlugin {
 	/**
+	 * Creates an instance of MemoryWithGcCachePlugin.
 	 * @param {MemoryWithGcCachePluginOptions} options options
 	 */
 	constructor({ maxGenerations }) {
+		/** @type {number} */
 		this._maxGenerations = maxGenerations;
 	}
 
 	/**
-	 * Apply the plugin
+	 * Applies the plugin by registering its hooks on the compiler.
 	 * @param {Compiler} compiler the compiler instance
 	 * @returns {void}
 	 */

package/lib/cache/PackFileCacheStrategy.js

@@ -7,9 +7,10 @@
 
 const FileSystemInfo = require("../FileSystemInfo");
 const ProgressPlugin = require("../ProgressPlugin");
-const { formatSize } = require("../SizeFormatHelpers");
+const { getReferencedFilenames } = require("../serialization/FileMiddleware");
 const SerializerMiddleware = require("../serialization/SerializerMiddleware");
 const LazySet = require("../util/LazySet");
+const formatSize = require("../util/formatSize");
 const makeSerializable = require("../util/makeSerializable");
 const memoize = require("../util/memoize");
 const {
@@ -34,9 +35,21 @@ const {
 /** @typedef {Set<string>} Items */
 /** @typedef {Set<string>} BuildDependencies */
 /** @typedef {Map<string, PackItemInfo>} ItemInfo */
+/** @typedef {{ firstSeen: number, size: number }} UnreferencedFile */
+/** @typedef {Map<string, UnreferencedFile>} UnreferencedFiles */
+
+// Unreferenced files are kept for this long to not race concurrent builds sharing the cache directory.
+const CLEANUP_GRACE_PERIOD = 30 * 60 * 1000;
+// Records when each unreferenced file was first seen. Aging by recorded time keeps
+// orphans expiring across caches restored with refreshed modification times.
+const UNREFERENCED_FILE = "unreferenced.json";
+// A file written this recently may belong to a concurrent build that reused the name,
+// in which case the recorded time describes the previous file and must not be trusted.
+const CLEANUP_RECENT_WRITE_PERIOD = 60 * 1000;
 
 class PackContainer {
 	/**
+	 * Creates an instance of PackContainer.
 	 * @param {Pack} data stored data
 	 * @param {string} version version identifier
 	 * @param {Snapshot} buildSnapshot snapshot of all build dependencies
@@ -67,6 +80,7 @@ class PackContainer {
 	}
 
 	/**
+	 * Serializes this instance into the provided serializer context.
 	 * @param {ObjectSerializerContext} context context
 	 */
 	serialize({ write, writeLazy }) {
@@ -80,6 +94,7 @@ class PackContainer {
 	}
 
 	/**
+	 * Restores this instance from the provided deserializer context.
 	 * @param {ObjectDeserializerContext} context context
 	 */
 	deserialize({ read }) {
@@ -102,10 +117,11 @@ const MIN_CONTENT_SIZE = 1024 * 1024; // 1 MB
 const CONTENT_COUNT_TO_MERGE = 10;
 const MIN_ITEMS_IN_FRESH_PACK = 100;
 const MAX_ITEMS_IN_FRESH_PACK = 50000;
-const MAX_TIME_IN_FRESH_PACK = 1 * 60 * 1000; // 1 min
+const MAX_TIME_IN_FRESH_PACK = 60 * 1000; // 1 min
 
 class PackItemInfo {
 	/**
+	 * Creates an instance of PackItemInfo.
 	 * @param {string} identifier identifier of item
 	 * @param {string | null | undefined} etag etag of item
 	 * @param {Data} value fresh value of item
@@ -126,6 +142,7 @@ class PackItemInfo {
 
 class Pack {
 	/**
+	 * Creates an instance of Pack.
 	 * @param {Logger} logger a logger
 	 * @param {number} maxAge max age of cache items
 	 */
@@ -149,6 +166,7 @@ class Pack {
 	}
 
 	/**
+	 * Adds the provided identifier to the pack.
 	 * @param {string} identifier identifier
 	 */
 	_addRequest(identifier) {
@@ -170,6 +188,7 @@ class Pack {
 	}
 
 	/**
+	 * Returns cached content.
 	 * @param {string} identifier unique name for the resource
 	 * @param {string | null} etag etag of the resource
 	 * @returns {Data} cached content
@@ -193,6 +212,7 @@ class Pack {
 	}
 
 	/**
+	 * Updates value using the provided identifier.
 	 * @param {string} identifier unique name for the resource
 	 * @param {string | null} etag etag of the resource
 	 * @param {Data} data cached content
@@ -244,6 +264,7 @@ class Pack {
 	}
 
 	/**
+	 * Returns new location of data entries.
 	 * @returns {number} new location of data entries
 	 */
 	_findLocation() {
@@ -254,6 +275,7 @@ class Pack {
 	}
 
 	/**
+	 * Gc and update location.
 	 * @private
 	 * @param {Items} items items
 	 * @param {Items} usedItems used items
@@ -559,6 +581,41 @@ class Pack {
 		}
 	}
 
+	/**
+	 * Drops every content whose items all expired. Unlike a partial collection this
+	 * never unpacks, so it is not limited to a single content per store and lets a
+	 * long unused cache shrink in one go instead of one pack per build.
+	 */
+	_gcExpiredContent() {
+		const now = Date.now();
+		let packCount = 0;
+		let itemCount = 0;
+		for (let loc = 0; loc < this.content.length; loc++) {
+			const content = this.content[loc];
+			if (!content) continue;
+			let expired = true;
+			for (const identifier of content.items) {
+				const info = this.itemInfo.get(identifier);
+				if (info !== undefined && now - info.lastAccess <= this.maxAge) {
+					expired = false;
+					break;
+				}
+			}
+			if (!expired) continue;
+			for (const identifier of content.items) this.itemInfo.delete(identifier);
+			this.content[loc] = undefined;
+			packCount++;
+			itemCount += content.items.size;
+		}
+		if (packCount > 0) {
+			this.logger.log(
+				"Garbage Collected %d completely expired packs with %d items",
+				packCount,
+				itemCount
+			);
+		}
+	}
+
 	/**
 	 * Find the content with the oldest item and run GC on that.
 	 * Only runs for one content to avoid large invalidation.
@@ -571,11 +628,10 @@ class Pack {
 				oldest = info;
 			}
 		}
-		if (
-			Date.now() - /** @type {PackItemInfo} */ (oldest).lastAccess >
-			this.maxAge
-		) {
-			const loc = /** @type {PackItemInfo} */ (oldest).location;
+		// collecting expired content may have left no items at all
+		if (oldest === undefined) return;
+		if (Date.now() - oldest.lastAccess > this.maxAge) {
+			const loc = oldest.location;
 			if (loc < 0) return;
 			const content = /** @type {PackContent} */ (this.content[loc]);
 			const items = new Set(content.items);
@@ -604,12 +660,14 @@ class Pack {
 	}
 
 	/**
+	 * Serializes this instance into the provided serializer context.
 	 * @param {ObjectSerializerContext} context context
 	 */
 	serialize({ write, writeSeparate }) {
 		this._persistFreshContent();
 		this._optimizeSmallContent();
 		this._optimizeUnusedContent();
+		this._gcExpiredContent();
 		this._gcOldestContent();
 		for (const identifier of this.itemInfo.keys()) {
 			write(identifier);
@@ -637,6 +695,7 @@ class Pack {
 	}
 
 	/**
+	 * Restores this instance from the provided deserializer context.
 	 * @param {ObjectDeserializerContext & { logger: Logger }} context context
 	 */
 	deserialize({ read, logger }) {
@@ -694,13 +753,16 @@ makeSerializable(Pack, "webpack/lib/cache/PackFileCacheStrategy", "Pack");
 
 class PackContentItems {
 	/**
+	 * Creates an instance of PackContentItems.
 	 * @param {Content} map items
 	 */
 	constructor(map) {
+		/** @type {Content} */
 		this.map = map;
 	}
 
 	/**
+	 * Serializes this instance into the provided serializer context.
 	 * @param {ObjectSerializerContext & { logger: Logger, profile: boolean | undefined }} context context
 	 */
 	serialize({ write, snapshot, rollback, logger, profile }) {
@@ -778,13 +840,14 @@ class PackContentItems {
 	}
 
 	/**
+	 * Restores this instance from the provided deserializer context.
 	 * @param {ObjectDeserializerContext & { logger: Logger, profile: boolean | undefined }} context context
 	 */
 	deserialize({ read, logger, profile }) {
 		if (read()) {
 			this.map = read();
 		} else if (profile) {
-			/** @type {Map<EXPECTED_ANY, EXPECTED_ANY>} */
+			/** @type {Content} */
 			const map = new Map();
 			let key = read();
 			while (key !== null) {
@@ -810,7 +873,7 @@ class PackContentItems {
 			}
 			this.map = map;
 		} else {
-			/** @type {Map<EXPECTED_ANY, EXPECTED_ANY>} */
+			/** @type {Content} */
 			const map = new Map();
 			let key = read();
 			while (key !== null) {
@@ -851,6 +914,7 @@ class PackContent {
 	*/
 
 	/**
+	 * Creates an instance of PackContent.
 	 * @param {Items} items keys
 	 * @param {Items} usedItems used keys
 	 * @param {PackContentItems | (() => Promise<PackContentItems>)} dataOrFn sync or async content
@@ -875,6 +939,7 @@ class PackContent {
 	}
 
 	/**
+	 * Returns result.
 	 * @param {string} identifier identifier
 	 * @returns {string | Promise<string>} result
 	 */
@@ -927,6 +992,7 @@ class PackContent {
 	}
 
 	/**
+	 * Returns maybe a promise if lazy.
 	 * @param {string} reason explanation why unpack is necessary
 	 * @returns {void | Promise<void>} maybe a promise if lazy
 	 */
@@ -971,6 +1037,7 @@ class PackContent {
 	}
 
 	/**
+	 * Returns the estimated size for the requested source type.
 	 * @returns {number} size of the content or -1 if not known
 	 */
 	getSize() {
@@ -985,6 +1052,7 @@ class PackContent {
 	}
 
 	/**
+	 * Processes the provided identifier.
 	 * @param {string} identifier identifier
 	 */
 	delete(identifier) {
@@ -994,6 +1062,7 @@ class PackContent {
 	}
 
 	/**
+	 * Processes the provided write.
 	 * @param {(lazy: LazyFunction) => (() => PackContentItems | Promise<PackContentItems>)} write write function
 	 * @returns {void}
 	 */
@@ -1086,6 +1155,7 @@ class PackContent {
 }
 
 /**
+ * Allow collecting memory.
  * @param {Buffer} buf buffer
  * @returns {Buffer} buffer that can be collected
  */
@@ -1099,6 +1169,7 @@ const allowCollectingMemory = (buf) => {
 
 class PackFileCacheStrategy {
 	/**
+	 * Creates an instance of PackFileCacheStrategy.
 	 * @param {object} options options
 	 * @param {Compiler} options.compiler the compiler
 	 * @param {IntermediateFileSystem} options.fs the filesystem
@@ -1110,7 +1181,7 @@ class PackFileCacheStrategy {
 	 * @param {number} options.maxAge max age of cache items
 	 * @param {boolean=} options.profile track and log detailed timing information for individual cache items
 	 * @param {boolean=} options.allowCollectingMemory allow to collect unused memory created during deserialization
-	 * @param {false | "gzip" | "brotli"=} options.compression compression used
+	 * @param {false | "gzip" | "brotli" | "zstd"=} options.compression compression used
 	 * @param {boolean=} options.readonly disable storing cache into filesystem
 	 */
 	constructor({
@@ -1142,6 +1213,8 @@ class PackFileCacheStrategy {
 		});
 		/** @type {Compiler} */
 		this.compiler = compiler;
+		/** @type {IntermediateFileSystem} */
+		this.fs = fs;
 		/** @type {string} */
 		this.context = context;
 		/** @type {string} */
@@ -1158,15 +1231,25 @@ class PackFileCacheStrategy {
 		this.readonly = readonly;
 		/** @type {boolean | undefined} */
 		this.allowCollectingMemory = allowCollectingMemory;
-		/** @type {false | "gzip" | "brotli" | undefined} */
+		// referenced names per on-disk file, versioned by mtime since concurrent
+		// processes may rewrite packs in place under the same name
+		/** @type {Map<string, { mtimeMs: number, referenced: string[] }>} */
+		this._referencedFilesCache = new Map();
+		/** @type {false | "gzip" | "brotli" | "zstd" | undefined} */
 		this.compression = compression;
+		// eslint-disable-next-line n/no-unsupported-features/node-builtins
+		if (compression === "zstd" && !require("zlib").createZstdCompress) {
+			throw new Error("cache.compression: 'zstd' requires Node.js >= 22.15.0");
+		}
 		/** @type {string} */
 		this._extension =
 			compression === "brotli"
 				? ".pack.br"
 				: compression === "gzip"
 					? ".pack.gz"
-					: ".pack";
+					: compression === "zstd"
+						? ".pack.zst"
+						: ".pack";
 		/** @type {SnapshotOptions} */
 		this.snapshot = snapshot;
 		/** @type {BuildDependencies} */
@@ -1186,6 +1269,7 @@ class PackFileCacheStrategy {
 	}
 
 	/**
+	 * Returns pack.
 	 * @returns {Promise<Pack>} pack
 	 */
 	_getPack() {
@@ -1196,6 +1280,7 @@ class PackFileCacheStrategy {
 	}
 
 	/**
+	 * Returns the pack.
 	 * @returns {Promise<Pack>} the pack
 	 */
 	_openPack() {
@@ -1362,6 +1447,7 @@ class PackFileCacheStrategy {
 	}
 
 	/**
+	 * Returns promise.
 	 * @param {string} identifier unique name for the resource
 	 * @param {Etag | null} etag etag of the resource
 	 * @param {Data} data cached content
@@ -1376,6 +1462,7 @@ class PackFileCacheStrategy {
 	}
 
 	/**
+	 * Returns promise to the cached content.
 	 * @param {string} identifier unique name for the resource
 	 * @param {Etag | null} etag etag of the resource
 	 * @returns {Promise<Data>} promise to the cached content
@@ -1396,6 +1483,7 @@ class PackFileCacheStrategy {
 	}
 
 	/**
+	 * Stores build dependencies.
 	 * @param {FileSystemDependencies | Iterable<string>} dependencies dependencies to store
 	 */
 	storeBuildDependencies(dependencies) {
@@ -1429,6 +1517,7 @@ class PackFileCacheStrategy {
 					);
 					promise = new Promise(
 						/**
+						 * Handles the callback logic for this hook.
 						 * @param {(value?: undefined) => void} resolve resolve
 						 * @param {(reason?: Error) => void} reject reject
 						 */
@@ -1552,10 +1641,17 @@ class PackFileCacheStrategy {
 						/** @type {Snapshot} */
 						(this.resolveBuildDependenciesSnapshot)
 					);
+					const cleanup = this.fs.unlink !== undefined;
+					/** @type {Set<string> | undefined} */
+					const writtenFiles = cleanup ? new Set() : undefined;
+					/** @type {Set<string> | undefined} */
+					const retainedFiles = cleanup ? new Set() : undefined;
 					return this.fileSerializer
 						.serialize(content, {
 							filename: `${this.cacheLocation}/index${this._extension}`,
 							extension: `${this._extension}`,
+							writtenFiles,
+							retainedFiles,
 							logger: this.logger,
 							profile: this.profile
 						})
@@ -1572,9 +1668,17 @@ class PackFileCacheStrategy {
 								stats.count,
 								Math.round(stats.size / 1024 / 1024)
 							);
+							if (writtenFiles !== undefined) {
+								return this._cleanupUnusedFiles(
+									writtenFiles,
+									/** @type {Set<string>} */ (retainedFiles)
+								);
+							}
 						})
 						.catch((err) => {
 							this.logger.timeEnd("store pack");
+							// files may be in an unknown state after a failed store
+							this._referencedFilesCache.clear();
 							this.logger.warn(`Caching failed for pack: ${err}`);
 							this.logger.debug(err.stack);
 						});
@@ -1586,6 +1690,192 @@ class PackFileCacheStrategy {
 			}));
 	}
 
+	/**
+	 * Reads when the currently unreferenced files were first seen. A missing or
+	 * unreadable file just restarts the grace period for every orphan.
+	 * @returns {Promise<UnreferencedFiles>} first seen time and size per file name
+	 */
+	_readUnreferencedFiles() {
+		return new Promise((resolve) => {
+			this.fs.readFile(
+				`${this.cacheLocation}/${UNREFERENCED_FILE}`,
+				(err, content) => {
+					/** @type {UnreferencedFiles} */
+					const result = new Map();
+					if (err) return resolve(result);
+					try {
+						const data = JSON.parse(
+							/** @type {Buffer} */ (content).toString("utf8")
+						);
+						for (const [file, entry] of Object.entries(data)) {
+							const { firstSeen, size } = /** @type {UnreferencedFile} */ (
+								entry
+							);
+							if (typeof firstSeen === "number" && typeof size === "number") {
+								result.set(file, { firstSeen, size });
+							}
+						}
+					} catch (_err) {
+						result.clear();
+					}
+					resolve(result);
+				}
+			);
+		});
+	}
+
+	/**
+	 * Persists when the still unreferenced files were first seen. Failing to write
+	 * only costs the orphans another grace period, so errors are ignored.
+	 * @param {UnreferencedFiles} unreferenced first seen time and size per file name
+	 * @param {boolean} hadEntries whether a previous state exists that must be replaced
+	 * @returns {Promise<void>} promise
+	 */
+	_writeUnreferencedFiles(unreferenced, hadEntries) {
+		if (unreferenced.size === 0 && !hadEntries) return Promise.resolve();
+		/** @type {Record<string, UnreferencedFile>} */
+		const data = {};
+		for (const [file, entry] of unreferenced) data[file] = entry;
+		return new Promise((resolve) => {
+			this.fs.writeFile(
+				`${this.cacheLocation}/${UNREFERENCED_FILE}`,
+				JSON.stringify(data),
+				() => resolve()
+			);
+		});
+	}
+
+	/**
+	 * Deletes files from the cache directory that are no longer referenced by the
+	 * stored pack. Retained files are walked on disk since nested lazy segments
+	 * reference files not visible during serialization. Errors only log a warning.
+	 * @param {Set<string>} writtenNames names (without extension) written by this store
+	 * @param {Set<string>} retainedNames names (without extension) referenced but not rewritten
+	 * @returns {Promise<void>} promise
+	 */
+	async _cleanupUnusedFiles(writtenNames, retainedNames) {
+		this.logger.time("cleanup unused cache files");
+		const fs = this.fs;
+		const extension = this._extension;
+		const cacheLocation = this.cacheLocation;
+		const referencedFilesCache = this._referencedFilesCache;
+		try {
+			// rewritten files may reference different names now
+			for (const name of writtenNames) referencedFilesCache.delete(name);
+			/** @type {Set<string>} */
+			const liveFiles = new Set([`index${extension}`, UNREFERENCED_FILE]);
+			for (const name of writtenNames) liveFiles.add(`${name}${extension}`);
+			/** @type {string[]} */
+			const queue = [];
+			/**
+			 * Marks a file live and queues it for walking its references.
+			 * @param {string} name file name without extension
+			 */
+			const enqueue = (name) => {
+				const file = `${name}${extension}`;
+				if (liveFiles.has(file)) return;
+				liveFiles.add(file);
+				queue.push(name);
+			};
+			for (const name of retainedNames) enqueue(name);
+			while (queue.length > 0) {
+				const name = /** @type {string} */ (queue.pop());
+				const file = `${cacheLocation}/${name}${extension}`;
+				// an unchanged mtime proves the memo entry still matches the disk
+				const mtimeMs = await new Promise((resolve, reject) => {
+					fs.stat(file, (err, stats) => {
+						if (err) return reject(err);
+						resolve(
+							/** @type {number} */ (
+								/** @type {import("../util/fs").IStats} */ (stats).mtimeMs
+							)
+						);
+					});
+				});
+				const entry = referencedFilesCache.get(name);
+				let referenced;
+				if (entry !== undefined && entry.mtimeMs === mtimeMs) {
+					referenced = entry.referenced;
+				} else {
+					referenced = await getReferencedFilenames(fs, file);
+					referencedFilesCache.set(name, { mtimeMs, referenced });
+				}
+				for (const referencedName of referenced) enqueue(referencedName);
+			}
+			const files = await new Promise((resolve, reject) => {
+				fs.readdir(cacheLocation, (err, files) => {
+					if (err) return reject(err);
+					resolve(/** @type {string[]} */ (files));
+				});
+			});
+			const seenFiles = await this._readUnreferencedFiles();
+			const now = Date.now();
+			// every store rewrites the index backup, so a recorded time would age a file
+			// that is in fact new; renaming carries the previous index mtime onto it
+			const indexBackup = `index${extension}.old`;
+			/** @type {UnreferencedFiles} */
+			const stillUnreferenced = new Map();
+			let deletedCount = 0;
+			for (const file of files) {
+				if (typeof file !== "string" || liveFiles.has(file)) continue;
+				const path = `${cacheLocation}/${file}`;
+				const stats = await new Promise((resolve) => {
+					fs.stat(path, (err, stats) => {
+						resolve(
+							err
+								? undefined
+								: /** @type {import("../util/fs").IStats} */ (stats)
+						);
+					});
+				});
+				if (stats === undefined || !stats.isFile()) continue;
+				const size = /** @type {number} */ (stats.size);
+				const seen = seenFiles.get(file);
+				// a differing size means the name was rewritten, so it is a new orphan
+				const firstSeen =
+					seen !== undefined && seen.size === size ? seen.firstSeen : now;
+				const expireTime = now - CLEANUP_GRACE_PERIOD;
+				const mtimeMs = /** @type {number} */ (stats.mtimeMs);
+				// either signal is enough: modification times are lost when a cache is
+				// restored, and recorded times are lost when the cache directory is new
+				const expired =
+					mtimeMs <= expireTime ||
+					(file !== indexBackup &&
+						firstSeen <= expireTime &&
+						mtimeMs <= now - CLEANUP_RECENT_WRITE_PERIOD);
+				if (expired) {
+					const deleted = await new Promise((resolve) => {
+						/** @type {NonNullable<IntermediateFileSystem["unlink"]>} */
+						(fs.unlink)(path, (err) => resolve(!err));
+					});
+					if (deleted) {
+						deletedCount++;
+						continue;
+					}
+				}
+				if (file !== indexBackup) {
+					stillUnreferenced.set(file, { firstSeen, size });
+				}
+			}
+			await this._writeUnreferencedFiles(stillUnreferenced, seenFiles.size > 0);
+			// drop entries of files that are no longer alive
+			for (const name of referencedFilesCache.keys()) {
+				if (!liveFiles.has(`${name}${extension}`)) {
+					referencedFilesCache.delete(name);
+				}
+			}
+			if (deletedCount > 0) {
+				this.logger.log("Deleted %d unused cache files", deletedCount);
+			}
+		} catch (err) {
+			this.logger.warn(
+				`Cleanup of unused cache files failed: ${/** @type {Error} */ (err)}`
+			);
+			this.logger.debug(/** @type {Error} */ (err).stack);
+		}
+		this.logger.timeEnd("cleanup unused cache files");
+	}
+
 	clear() {
 		this.fileSystemInfo.clear();
 		this.buildDependencies.clear();