@depup/webpack 5.109.0-depup.0 → 5.109.2-depup.0

package/lib/serialization/FileMiddleware.js

@@ -7,6 +7,8 @@
 const { constants } = require("buffer");
 const { pipeline } = require("stream");
 const {
+	// eslint-disable-next-line n/no-unsupported-features/node-builtins
+	brotliDecompress,
 	constants: zConstants,
 	// eslint-disable-next-line n/no-unsupported-features/node-builtins
 	createBrotliCompress,
@@ -18,7 +20,10 @@ const {
 	// eslint-disable-next-line n/no-unsupported-features/node-builtins
 	createZstdCompress,
 	// eslint-disable-next-line n/no-unsupported-features/node-builtins
-	createZstdDecompress
+	createZstdDecompress,
+	gunzip,
+	// eslint-disable-next-line n/no-unsupported-features/node-builtins
+	zstdDecompress
 } = require("zlib");
 const { DEFAULTS } = require("../config/defaults");
 const createHash = require("../util/createHash");
@@ -51,6 +56,8 @@ Section -> Buffer
 const VERSION = 0x01637077;
 const WRITE_LIMIT_TOTAL = 0x7fff0000;
 const WRITE_LIMIT_CHUNK = 511 * 1024 * 1024;
+// headers and pointer sections are tiny; anything above this is a corrupt file
+const MAX_HEADER_OR_POINTER_SIZE = 256 * 1024 * 1024;
 
 /**
  * Returns hash.
@@ -111,6 +118,7 @@ const readUInt64LE = Buffer.prototype.readBigUInt64LE
  * @param {string | boolean} name file base name
  * @param {(name: string | false, buffers: Buffer[], size: number) => Promise<void>} writeFile writes a file
  * @param {HashFunction=} hashFunction hash function to use
+ * @param {Set<string>=} retainedNames collects names of files that stay referenced without being rewritten
  * @returns {Promise<SerializeResult>} resulting file pointer and promise
  */
 const serialize = async (
@@ -118,7 +126,8 @@ const serialize = async (
 	data,
 	name,
 	writeFile,
-	hashFunction = DEFAULTS.HASH_FUNCTION
+	hashFunction = DEFAULTS.HASH_FUNCTION,
+	retainedNames = undefined
 ) => {
 	/** @type {(Buffer[] | Buffer | Promise<SerializeResult>)[]} */
 	const processedData = [];
@@ -144,6 +153,10 @@ const serialize = async (
 						"Unexpected lazy value with non-this target (can't pass through lazy values)"
 					);
 				} else {
+					if (retainedNames !== undefined) {
+						// pointer buffer layout: u64 size + utf-8 file name
+						retainedNames.add(serializedInfo.toString("utf8", 8));
+					}
 					processedData.push(serializedInfo);
 				}
 			} else {
@@ -157,7 +170,8 @@ const serialize = async (
 							(content),
 							(options && options.name) || true,
 							writeFile,
-							hashFunction
+							hashFunction,
+							retainedNames
 						).then((result) => {
 							/** @type {LazyOptions} */
 							(item.options).size = result.size;
@@ -444,7 +458,14 @@ const deserialize = async (middleware, name, readFile) => {
 
 /** @typedef {BufferSerializableType[]} DeserializedType */
 /** @typedef {true} SerializedType */
-/** @typedef {{ filename: string, extension?: string }} Context */
+/**
+ * `writtenFiles`/`retainedFiles` collect file names (without extension) during
+ * `serialize`: files written in this run and files that stay referenced by
+ * lazy pointers without being rewritten. Retained files may reference further
+ * files on disk not listed here (nested lazy segments); use
+ * `getReferencedFilenames` to walk them.
+ * @typedef {{ filename: string, extension?: string, writtenFiles?: Set<string>, retainedFiles?: Set<string> }} Context
+ */
 
 /**
  * Represents FileMiddleware.
@@ -471,7 +492,7 @@ class FileMiddleware extends SerializerMiddleware {
 	 * @returns {SerializedType | Promise<SerializedType> | null} serialized data
 	 */
 	serialize(data, context) {
-		const { filename, extension = "" } = context;
+		const { filename, extension = "", writtenFiles, retainedFiles } = context;
 		return new Promise((resolve, reject) => {
 			mkdirp(this.fs, dirname(this.fs, filename), (err) => {
 				if (err) return reject(err);
@@ -581,66 +602,74 @@ class FileMiddleware extends SerializerMiddleware {
 							batchWrite();
 						}
 					);
-					if (name) allWrittenFiles.add(file);
+					if (name) {
+						allWrittenFiles.add(file);
+						if (writtenFiles !== undefined) writtenFiles.add(name);
+					}
 				};
 
 				resolve(
-					serialize(this, data, false, writeFile, this._hashFunction).then(
-						async ({ backgroundJob }) => {
-							await backgroundJob;
-
-							// Rename the index file to disallow access during inconsistent file state
-							await new Promise(
-								/**
-								 * Handles the callback logic for this hook.
-								 * @param {(value?: undefined) => void} resolve resolve
-								 */
-								(resolve) => {
-									this.fs.rename(filename, `${filename}.old`, (_err) => {
-										resolve();
-									});
-								}
-							);
+					serialize(
+						this,
+						data,
+						false,
+						writeFile,
+						this._hashFunction,
+						retainedFiles
+					).then(async ({ backgroundJob }) => {
+						await backgroundJob;
 
-							// update all written files
-							await Promise.all(
-								Array.from(
-									allWrittenFiles,
-									(file) =>
-										new Promise(
-											/**
-											 * Handles the callback logic for this hook.
-											 * @param {(value?: undefined) => void} resolve resolve
-											 * @param {(reason?: Error | null) => void} reject reject
-											 * @returns {void}
-											 */
-											(resolve, reject) => {
-												this.fs.rename(`${file}_`, file, (err) => {
-													if (err) return reject(err);
-													resolve();
-												});
-											}
-										)
-								)
-							);
+						// Rename the index file to disallow access during inconsistent file state
+						await new Promise(
+							/**
+							 * Handles the callback logic for this hook.
+							 * @param {(value?: undefined) => void} resolve resolve
+							 */
+							(resolve) => {
+								this.fs.rename(filename, `${filename}.old`, (_err) => {
+									resolve();
+								});
+							}
+						);
 
-							// As final step automatically update the index file to have a consistent pack again
-							await new Promise(
-								/**
-								 * Handles the callback logic for this hook.
-								 * @param {(value?: undefined) => void} resolve resolve
-								 * @returns {void}
-								 */
-								(resolve) => {
-									this.fs.rename(`${filename}_`, filename, (err) => {
-										if (err) return reject(err);
-										resolve();
-									});
-								}
-							);
-							return /** @type {true} */ (true);
-						}
-					)
+						// update all written files
+						await Promise.all(
+							Array.from(
+								allWrittenFiles,
+								(file) =>
+									new Promise(
+										/**
+										 * Handles the callback logic for this hook.
+										 * @param {(value?: undefined) => void} resolve resolve
+										 * @param {(reason?: Error | null) => void} reject reject
+										 * @returns {void}
+										 */
+										(resolve, reject) => {
+											this.fs.rename(`${file}_`, file, (err) => {
+												if (err) return reject(err);
+												resolve();
+											});
+										}
+									)
+							)
+						);
+
+						// As final step automatically update the index file to have a consistent pack again
+						await new Promise(
+							/**
+							 * Handles the callback logic for this hook.
+							 * @param {(value?: undefined) => void} resolve resolve
+							 * @returns {void}
+							 */
+							(resolve) => {
+								this.fs.rename(`${filename}_`, filename, (err) => {
+									if (err) return reject(err);
+									resolve();
+								});
+							}
+						);
+						return /** @type {true} */ (true);
+					})
 				);
 			});
 		});
@@ -808,6 +837,216 @@ class FileMiddleware extends SerializerMiddleware {
 	}
 }
 
+/**
+ * Extracts the file names referenced by lazy pointer sections from serialized content.
+ * @param {Buffer} buf decompressed file content
+ * @returns {string[]} referenced file names (without extension)
+ */
+const parsePointerNames = (buf) => {
+	const version = buf.readUInt32LE(0);
+	if (version !== VERSION) {
+		throw new Error(`Invalid file version ${version}`);
+	}
+	const sectionCount = buf.readUInt32LE(4);
+	let offset = 8 + sectionCount * 4;
+	// a corrupt section table must abort the walk, never silently drop a name
+	if (offset > buf.length) {
+		throw new Error(`Invalid section count ${sectionCount}`);
+	}
+	/** @type {string[]} */
+	const names = [];
+	for (let i = 0; i < sectionCount; i++) {
+		const length = buf.readInt32LE(8 + i * 4);
+		if (length < 0) {
+			// pointer section: u64 size + utf-8 file name
+			const end = offset - length;
+			if (end > buf.length) {
+				throw new Error("Truncated pointer section");
+			}
+			names.push(buf.toString("utf8", offset + 8, end));
+			offset = end;
+		} else {
+			offset += length;
+		}
+	}
+	if (offset !== buf.length) {
+		throw new Error("Section table does not match file size");
+	}
+	return names;
+};
+
+/**
+ * Reads the pointer names of a compressed file by decompressing it fully
+ * (compressed content cannot be read by byte range).
+ * @param {IntermediateFileSystem} fs a file system
+ * @param {string} file absolute path of the serialized file
+ * @returns {Promise<string[]>} referenced file names (without extension)
+ */
+const getReferencedFilenamesCompressed = (fs, file) =>
+	new Promise((resolve, reject) => {
+		fs.readFile(file, (err, rawContent) => {
+			if (err) return reject(err);
+			/**
+			 * Parses the decompressed content.
+			 * @param {Error | null} err error
+			 * @param {Buffer=} content decompressed content
+			 * @returns {void}
+			 */
+			const onContent = (err, content) => {
+				if (err) return reject(err);
+				try {
+					resolve(parsePointerNames(/** @type {Buffer} */ (content)));
+				} catch (err_) {
+					reject(/** @type {Error} */ (err_));
+				}
+			};
+			const buf = /** @type {Buffer} */ (rawContent);
+			if (file.endsWith(".gz")) {
+				gunzip(buf, onContent);
+			} else if (file.endsWith(".br")) {
+				brotliDecompress(buf, onContent);
+			} else {
+				if (!zstdDecompress) {
+					return reject(
+						new Error("zstd decompression requires Node.js >= 22.15.0")
+					);
+				}
+				zstdDecompress(buf, onContent);
+			}
+		});
+	});
+
+/**
+ * Reads the pointer names of an uncompressed file by reading only the header
+ * and the pointer sections instead of the whole file.
+ * @param {IntermediateFileSystem} fs a file system
+ * @param {string} file absolute path of the serialized file
+ * @returns {Promise<string[]>} referenced file names (without extension)
+ */
+const getReferencedFilenamesUncompressed = (fs, file) =>
+	new Promise((resolve, reject) => {
+		fs.open(file, "r", (err, _fd) => {
+			if (err) return reject(err);
+			const fd = /** @type {number} */ (_fd);
+			/**
+			 * Closes the file descriptor and rejects.
+			 * @param {Error} err error
+			 */
+			const fail = (err) => {
+				fs.close(fd, () => reject(err));
+			};
+			/**
+			 * Reads exactly `length` bytes at `position`.
+			 * @param {number} position file position
+			 * @param {number} length byte count
+			 * @param {(buffer: Buffer) => void} callback called with the filled buffer
+			 * @returns {void}
+			 */
+			const readAt = (position, length, callback) => {
+				// corrupt headers can request absurd sizes; must reject, not crash
+				if (length > MAX_HEADER_OR_POINTER_SIZE) {
+					return fail(new Error(`Invalid section size ${length} in ${file}`));
+				}
+				/** @type {Buffer} */
+				let buffer;
+				try {
+					buffer = Buffer.allocUnsafe(length);
+				} catch (err) {
+					return fail(/** @type {Error} */ (err));
+				}
+				let bytesDone = 0;
+				const readMore = () => {
+					fs.read(
+						fd,
+						buffer,
+						bytesDone,
+						length - bytesDone,
+						position + bytesDone,
+						(err, bytesRead) => {
+							if (err) return fail(err);
+							if (bytesRead === 0) {
+								return fail(new Error(`Unexpected end of file ${file}`));
+							}
+							bytesDone += bytesRead;
+							if (bytesDone < length) return readMore();
+							callback(buffer);
+						}
+					);
+				};
+				readMore();
+			};
+			readAt(0, 8, (header) => {
+				const version = header.readUInt32LE(0);
+				if (version !== VERSION) {
+					return fail(new Error(`Invalid file version ${version}`));
+				}
+				const sectionCount = header.readUInt32LE(4);
+				if (sectionCount === 0) {
+					return fs.close(fd, (err) => (err ? reject(err) : resolve([])));
+				}
+				readAt(8, sectionCount * 4, (lengthsBuffer) => {
+					/** @type {{ position: number, length: number }[]} */
+					const pointerSections = [];
+					let position = 8 + sectionCount * 4;
+					for (let i = 0; i < sectionCount; i++) {
+						const length = lengthsBuffer.readInt32LE(i * 4);
+						if (length < 0) {
+							pointerSections.push({ position, length: -length });
+							position -= length;
+						} else {
+							position += length;
+						}
+					}
+					// the section table must account for exactly the whole file,
+					// otherwise pointer reads land on wrong bytes and under-report
+					fs.stat(file, (err, stats) => {
+						if (err) return fail(err);
+						const stat = /** @type {import("../util/fs").IStats} */ (stats);
+						if (stat.size !== position) {
+							return fail(
+								new Error(`Section table does not match size of ${file}`)
+							);
+						}
+						/** @type {string[]} */
+						const names = [];
+						/**
+						 * Reads the pointer section at `index`, then the next one.
+						 * @param {number} index pointer section index
+						 * @returns {void}
+						 */
+						const readNext = (index) => {
+							if (index >= pointerSections.length) {
+								return fs.close(fd, (err) =>
+									err ? reject(err) : resolve(names)
+								);
+							}
+							const section = pointerSections[index];
+							readAt(section.position, section.length, (buffer) => {
+								// pointer section: u64 size + utf-8 file name
+								names.push(buffer.toString("utf8", 8));
+								readNext(index + 1);
+							});
+						};
+						readNext(0);
+					});
+				});
+			});
+		});
+	});
+
+/**
+ * Reads the file names referenced by a serialized file (its lazy pointer sections)
+ * without deserializing its content. Referenced files may reference further files.
+ * @param {IntermediateFileSystem} fs a file system
+ * @param {string} file absolute path of the serialized file
+ * @returns {Promise<string[]>} referenced file names (without extension)
+ */
+const getReferencedFilenames = (fs, file) =>
+	file.endsWith(".gz") || file.endsWith(".br") || file.endsWith(".zst")
+		? getReferencedFilenamesCompressed(fs, file)
+		: getReferencedFilenamesUncompressed(fs, file);
+
 module.exports = FileMiddleware;
 // Exposed for testing the content-buffer boundary handling in `deserialize`.
 module.exports._deserialize = deserialize;
+module.exports.getReferencedFilenames = getReferencedFilenames;