@depup/webpack 5.106.1-depup.0 → 5.109.2-depup.0

package/lib/optimize/ModuleConcatenationPlugin.js

@@ -7,10 +7,14 @@
 
 const asyncLib = require("neo-async");
 const ChunkGraph = require("../ChunkGraph");
+const Dependency = require("../Dependency");
+const Module = require("../Module");
 const ModuleGraph = require("../ModuleGraph");
 const { JAVASCRIPT_TYPE } = require("../ModuleSourceTypeConstants");
 const { STAGE_DEFAULT } = require("../OptimizationStages");
+const CommonJsSelfReferenceDependency = require("../dependencies/CommonJsSelfReferenceDependency");
 const HarmonyImportDependency = require("../dependencies/HarmonyImportDependency");
+const { ImportPhaseUtils } = require("../dependencies/ImportPhase");
 const { compareModulesByIdentifier } = require("../util/comparators");
 const {
 	filterRuntime,
@@ -21,16 +25,38 @@ const {
 } = require("../util/runtime");
 const ConcatenatedModule = require("./ConcatenatedModule");
 
+/** @typedef {import("../Chunk")} Chunk */
 /** @typedef {import("../Compilation")} Compilation */
 /** @typedef {import("../Compiler")} Compiler */
-/** @typedef {import("../Module")} Module */
 /** @typedef {import("../Module").BuildInfo} BuildInfo */
+/** @typedef {import("../Module").BuildMeta} BuildMeta */
 /** @typedef {import("../RequestShortener")} RequestShortener */
 /** @typedef {import("../util/runtime").RuntimeSpec} RuntimeSpec */
 
 /** @typedef {Module | ((requestShortener: RequestShortener) => string)} Problem */
 
 /**
+ * Merged runtime of all chunks a module is in, memoized for the whole pass.
+ * Safe: the chunk graph is not mutated while configurations are built.
+ * @param {ChunkGraph} chunkGraph the chunk graph
+ * @param {Map<Module, RuntimeSpec>} cache memoization cache
+ * @param {Module} module the module
+ * @returns {RuntimeSpec} the merged runtime
+ */
+const getMergedModuleRuntime = (chunkGraph, cache, module) => {
+	const cached = cache.get(module);
+	if (cached !== undefined || cache.has(module)) return cached;
+	/** @type {RuntimeSpec} */
+	let runtime;
+	for (const r of chunkGraph.getModuleRuntimes(module)) {
+		runtime = mergeRuntimeOwned(runtime, r);
+	}
+	cache.set(module, runtime);
+	return runtime;
+};
+
+/**
+ * Defines the statistics type used by this module.
  * @typedef {object} Statistics
  * @property {number} cached
  * @property {number} alreadyInConfig
@@ -45,6 +71,24 @@ const ConcatenatedModule = require("./ConcatenatedModule");
  */
 
 /**
+ * Record why `module` can't be added (so retried lookups hit the cache), bump
+ * the matching statistic, and return the problem to bubble up the recursion.
+ * Only reached on failure, so it stays off the hot success path.
+ * @param {Map<Module, Problem>} failureCache per-module failure cache
+ * @param {Statistics} statistics running statistics
+ * @param {keyof Statistics} statKey statistic to increment
+ * @param {Module} module the module that couldn't be added
+ * @param {Problem} problem the failure to cache and return
+ * @returns {Problem} the same problem
+ */
+const cacheFailure = (failureCache, statistics, statKey, module, problem) => {
+	statistics[statKey]++;
+	failureCache.set(module, problem);
+	return problem;
+};
+
+/**
+ * Format bailout reason.
  * @param {string} msg message
  * @returns {string} formatted message
  */
@@ -54,7 +98,7 @@ const PLUGIN_NAME = "ModuleConcatenationPlugin";
 
 class ModuleConcatenationPlugin {
 	/**
-	 * Apply the plugin
+	 * Applies the plugin by registering its hooks on the compiler.
 	 * @param {Compiler} compiler the compiler instance
 	 * @returns {void}
 	 */
@@ -71,6 +115,7 @@ class ModuleConcatenationPlugin {
 			const bailoutReasonMap = new Map();
 
 			/**
+			 * Sets bailout reason.
 			 * @param {Module} module the module
 			 * @param {string | ((requestShortener: RequestShortener) => string)} reason the reason
 			 */
@@ -86,6 +131,7 @@ class ModuleConcatenationPlugin {
 			};
 
 			/**
+			 * Sets inner bailout reason.
 			 * @param {Module} module the module
 			 * @param {string | ((requestShortener: RequestShortener) => string)} reason the reason
 			 */
@@ -94,6 +140,7 @@ class ModuleConcatenationPlugin {
 			};
 
 			/**
+			 * Gets inner bailout reason.
 			 * @param {Module} module the module
 			 * @param {RequestShortener} requestShortener the request shortener
 			 * @returns {string | ((requestShortener: RequestShortener) => string) | undefined} the reason
@@ -105,6 +152,7 @@ class ModuleConcatenationPlugin {
 			};
 
 			/**
+			 * Format bailout warning.
 			 * @param {Module} module the module
 			 * @param {Problem} problem the problem
 			 * @returns {(requestShortener: RequestShortener) => string} the reason
@@ -149,9 +197,16 @@ class ModuleConcatenationPlugin {
 					const relevantModules = [];
 					/** @type {Set<Module>} */
 					const possibleInners = new Set();
+					const concatenateModules =
+						compilation.options.optimization.concatenateModules;
 					const context = {
 						chunkGraph,
-						moduleGraph
+						moduleGraph,
+						// commonjs concatenation defaults to on; { commonjs: false } opts out
+						concatenateCommonJsModules:
+							typeof concatenateModules === "object"
+								? concatenateModules.commonjs !== false
+								: concatenateModules === true
 					};
 					const deferEnabled = compilation.options.experiments.deferImport;
 					logger.time("select relevant modules");
@@ -170,7 +225,6 @@ class ModuleConcatenationPlugin {
 							setBailoutReason(module, "Module is async");
 							continue;
 						}
-
 						// Must be in strict mode
 						if (!(/** @type {BuildInfo} */ (module.buildInfo).strict)) {
 							setBailoutReason(module, "Module is not in strict mode");
@@ -222,6 +276,22 @@ class ModuleConcatenationPlugin {
 							canBeRoot = false;
 						}
 
+						// TODO: ConcatenatedModule.getSourceTypes only javascript now
+						const basicTypes = Module.getSourceBasicTypes(module);
+						if (basicTypes.size !== 1 || !basicTypes.has(JAVASCRIPT_TYPE)) {
+							canBeRoot = false;
+						}
+
+						// A concatenated CommonJS module relies on the root's ESM export
+						// rendering, so it can only be an inner module
+						if (
+							module.type.startsWith("javascript/") &&
+							/** @type {BuildMeta} */
+							(module.buildMeta).exportsType !== "namespace"
+						) {
+							canBeRoot = false;
+						}
+
 						// Module must not be an entry point
 						if (chunkGraph.isEntryModule(module)) {
 							setInnerBailoutReason(module, "Module is an entry point");
@@ -273,17 +343,20 @@ class ModuleConcatenationPlugin {
 					const concatConfigurations = [];
 					/** @type {Set<Module>} */
 					const usedAsInner = new Set();
+					// module -> merged runtime of its chunks, shared across all roots
+					/** @type {Map<Module, RuntimeSpec>} */
+					const moduleRuntimeCache = new Map();
 					for (const currentRoot of relevantModules) {
 						// when used by another configuration as inner:
 						// the other configuration is better and we can skip this one
 						// TODO reconsider that when it's only used in a different runtime
 						if (usedAsInner.has(currentRoot)) continue;
 
-						/** @type {RuntimeSpec} */
-						let chunkRuntime;
-						for (const r of chunkGraph.getModuleRuntimes(currentRoot)) {
-							chunkRuntime = mergeRuntimeOwned(chunkRuntime, r);
-						}
+						const chunkRuntime = getMergedModuleRuntime(
+							chunkGraph,
+							moduleRuntimeCache,
+							currentRoot
+						);
 						const exportsInfo = moduleGraph.getExportsInfo(currentRoot);
 						const filteredRuntime = filterRuntime(chunkRuntime, (r) =>
 							exportsInfo.isModuleUsed(r)
@@ -295,10 +368,12 @@ class ModuleConcatenationPlugin {
 									? undefined
 									: filteredRuntime;
 
-						// create a configuration with the root
+						// create a configuration with the root; cache the root's chunks once
+						// so every candidate check reuses them instead of re-materializing
 						const currentConfiguration = new ConcatConfiguration(
 							currentRoot,
-							activeRuntime
+							activeRuntime,
+							[...chunkGraph.getModuleChunksIterable(currentRoot)]
 						);
 
 						// cache failures to add modules
@@ -330,6 +405,7 @@ class ModuleConcatenationPlugin {
 								possibleInners,
 								impCandidates,
 								failureCache,
+								moduleRuntimeCache,
 								chunkGraph,
 								true,
 								stats
@@ -375,10 +451,10 @@ class ModuleConcatenationPlugin {
 					logger.debug(
 						`${statsCandidates} candidates were considered for adding (${stats.cached} cached failure, ${stats.alreadyInConfig} already in config, ${stats.invalidModule} invalid module, ${stats.incorrectChunks} incorrect chunks, ${stats.incorrectDependency} incorrect dependency, ${stats.incorrectChunksOfImporter} incorrect chunks of importer, ${stats.incorrectModuleDependency} incorrect module dependency, ${stats.incorrectRuntimeCondition} incorrect runtime condition, ${stats.importerFailed} importer failed, ${stats.added} added)`
 					);
-					// HACK: Sort configurations by length and start with the longest one
-					// to get the biggest groups possible. Used modules are marked with usedModules
-					// TODO: Allow to reuse existing configuration while trying to add dependencies.
-					// This would improve performance. O(n^2) -> O(n)
+					// Create the largest configurations first: if a root also ended up inside
+					// a bigger configuration, the bigger one wins (smaller skipped below).
+					// Configurations can't be reused across roots: a module shared by several
+					// chunks is concatenated into each consumer (module duplication).
 					logger.time("sort concat configurations");
 					concatConfigurations.sort((a, b) => b.modules.size - a.modules.size);
 					logger.timeEnd("sort concat configurations");
@@ -391,8 +467,8 @@ class ModuleConcatenationPlugin {
 						(concatConfiguration, callback) => {
 							const rootModule = concatConfiguration.rootModule;
 
-							// Avoid overlapping configurations
-							// TODO: remove this when todo above is fixed
+							// Root already concatenated into a larger configuration: skip.
+							// (Inner modules may still be shared, e.g. duplicated across entries.)
 							if (usedModules.has(rootModule)) return callback();
 							const modules = concatConfiguration.getModules();
 							for (const m of modules) {
@@ -454,7 +530,8 @@ class ModuleConcatenationPlugin {
 											(c) =>
 												c.originModule === m &&
 												!(
-													c.dependency instanceof HarmonyImportDependency &&
+													c.dependency &&
+													Dependency.canConcatenate(c.dependency) &&
 													modules.has(c.module)
 												)
 										);
@@ -497,7 +574,8 @@ class ModuleConcatenationPlugin {
 										const otherModule =
 											c.module === rootModule ? c.originModule : c.module;
 										const innerConnection =
-											c.dependency instanceof HarmonyImportDependency &&
+											c.dependency &&
+											Dependency.canConcatenate(c.dependency) &&
 											modules.has(/** @type {Module} */ (otherModule));
 										return !innerConnection;
 									}
@@ -521,6 +599,27 @@ class ModuleConcatenationPlugin {
 	}
 
 	/**
+	 * Checks whether the module `import defer`s a module already in the config.
+	 * @param {ModuleGraph} moduleGraph the module graph
+	 * @param {Module} module the candidate module
+	 * @param {ConcatConfiguration} config the concat configuration
+	 * @returns {boolean} true when a deferred import targets a module in the config
+	 */
+	_defersModuleIn(moduleGraph, module, config) {
+		for (const dep of module.dependencies) {
+			if (
+				dep instanceof HarmonyImportDependency &&
+				ImportPhaseUtils.isDefer(dep.phase)
+			) {
+				const target = moduleGraph.getModule(dep);
+				if (target && config.has(target)) return true;
+			}
+		}
+		return false;
+	}
+
+	/**
+	 * Returns the imported modules.
 	 * @param {Compilation} compilation the compilation
 	 * @param {Module} module the module to be added
 	 * @param {RuntimeSpec} runtime the runtime scope
@@ -531,8 +630,8 @@ class ModuleConcatenationPlugin {
 		/** @type {Set<Module>} */
 		const set = new Set();
 		for (const dep of module.dependencies) {
-			// Get reference info only for harmony Dependencies
-			if (!(dep instanceof HarmonyImportDependency)) continue;
+			// Get reference info only for dependencies that support concatenation
+			if (!Dependency.canConcatenate(dep)) continue;
 
 			const connection = moduleGraph.getConnection(dep);
 			// Reference is valid and has a module
@@ -562,6 +661,7 @@ class ModuleConcatenationPlugin {
 	}
 
 	/**
+	 * Returns the problematic module.
 	 * @param {Compilation} compilation webpack compilation
 	 * @param {ConcatConfiguration} config concat configuration (will be modified when added)
 	 * @param {Module} module the module to be added
@@ -570,6 +670,7 @@ class ModuleConcatenationPlugin {
 	 * @param {Set<Module>} possibleModules modules that are candidates
 	 * @param {Set<Module>} candidates list of potential candidates (will be added to)
 	 * @param {Map<Module, Problem>} failureCache cache for problematic modules to be more performant
+	 * @param {Map<Module, RuntimeSpec>} moduleRuntimeCache memoized merged runtime per module, shared across roots
 	 * @param {ChunkGraph} chunkGraph the chunk graph
 	 * @param {boolean} avoidMutateOnFailure avoid mutating the config when adding fails
 	 * @param {Statistics} statistics gathering metrics
@@ -584,6 +685,7 @@ class ModuleConcatenationPlugin {
 		possibleModules,
 		candidates,
 		failureCache,
+		moduleRuntimeCache,
 		chunkGraph,
 		avoidMutateOnFailure,
 		statistics
@@ -602,17 +704,46 @@ class ModuleConcatenationPlugin {
 
 		// Not possible to add?
 		if (!possibleModules.has(module)) {
-			statistics.invalidModule++;
-			failureCache.set(module, module); // cache failures for performance
-			return module;
+			return cacheFailure(
+				failureCache,
+				statistics,
+				"invalidModule",
+				module,
+				module
+			);
+		}
+
+		// A deferred import must stay a concatenation boundary: concatenating a
+		// module with a module it `import defer`s (a cycle) would erase the
+		// deferred namespace's lazy-evaluation and exotic-object semantics.
+		if (
+			compilation.options.experiments.deferImport &&
+			this._defersModuleIn(compilation.moduleGraph, module, config)
+		) {
+			/**
+			 * @param {RequestShortener} requestShortener request shortener
+			 * @returns {string} problem description
+			 */
+			const problem = (requestShortener) =>
+				`Module ${module.readableIdentifier(
+					requestShortener
+				)} imports a module in this configuration via import defer`;
+			return cacheFailure(
+				failureCache,
+				statistics,
+				"incorrectDependency",
+				module,
+				problem
+			);
 		}
 
 		// Module must be in the correct chunks
-		const missingChunks = [
-			...chunkGraph.getModuleChunksIterable(config.rootModule)
-		].filter((chunk) => !chunkGraph.isModuleInChunk(module, chunk));
+		const missingChunks = config.rootChunks.filter(
+			(chunk) => !chunkGraph.isModuleInChunk(module, chunk)
+		);
 		if (missingChunks.length > 0) {
 			/**
+			 * Returns problem description.
 			 * @param {RequestShortener} requestShortener request shortener
 			 * @returns {string} problem description
 			 */
@@ -635,9 +766,13 @@ class ModuleConcatenationPlugin {
 					", "
 				)}, module is in chunk(s) ${chunks.join(", ")})`;
 			};
-			statistics.incorrectChunks++;
-			failureCache.set(module, problem); // cache failures for performance
-			return problem;
+			return cacheFailure(
+				failureCache,
+				statistics,
+				"incorrectChunks",
+				module,
+				problem
+			);
 		}
 
 		const moduleGraph = compilation.moduleGraph;
@@ -656,6 +791,7 @@ class ModuleConcatenationPlugin {
 				);
 			if (activeNonModulesConnections.length > 0) {
 				/**
+				 * Returns problem description.
 				 * @param {RequestShortener} requestShortener request shortener
 				 * @returns {string} problem description
 				 */
@@ -675,9 +811,13 @@ class ModuleConcatenationPlugin {
 							: "in an unsupported way"
 					}`;
 				};
-				statistics.incorrectDependency++;
-				failureCache.set(module, problem); // cache failures for performance
-				return problem;
+				return cacheFailure(
+					failureCache,
+					statistics,
+					"incorrectDependency",
+					module,
+					problem
+				);
 			}
 		}
 
@@ -689,11 +829,11 @@ class ModuleConcatenationPlugin {
 				if (chunkGraph.getNumberOfModuleChunks(originModule) === 0) continue;
 
 				// We don't care for connections from other runtimes
-				/** @type {RuntimeSpec} */
-				let originRuntime;
-				for (const r of chunkGraph.getModuleRuntimes(originModule)) {
-					originRuntime = mergeRuntimeOwned(originRuntime, r);
-				}
+				const originRuntime = getMergedModuleRuntime(
+					chunkGraph,
+					moduleRuntimeCache,
+					originModule
+				);
 
 				if (!intersectRuntime(runtime, originRuntime)) continue;
 
@@ -711,9 +851,7 @@ class ModuleConcatenationPlugin {
 
 		// Module must be in the same chunks like the referencing module
 		const otherChunkModules = incomingModules.filter((originModule) => {
-			for (const chunk of chunkGraph.getModuleChunksIterable(
-				config.rootModule
-			)) {
+			for (const chunk of config.rootChunks) {
 				if (!chunkGraph.isModuleInChunk(originModule, chunk)) {
 					return true;
 				}
@@ -722,6 +860,7 @@ class ModuleConcatenationPlugin {
 		});
 		if (otherChunkModules.length > 0) {
 			/**
+			 * Returns problem description.
 			 * @param {RequestShortener} requestShortener request shortener
 			 * @returns {string} problem description
 			 */
@@ -735,9 +874,13 @@ class ModuleConcatenationPlugin {
 					", "
 				)}`;
 			};
-			statistics.incorrectChunksOfImporter++;
-			failureCache.set(module, problem); // cache failures for performance
-			return problem;
+			return cacheFailure(
+				failureCache,
+				statistics,
+				"incorrectChunksOfImporter",
+				module,
+				problem
+			);
 		}
 
 		/** @type {Map<Module, ReadonlyArray<ModuleGraph.ModuleGraphConnection>>} */
@@ -746,7 +889,13 @@ class ModuleConcatenationPlugin {
 			const selected = connections.filter(
 				(connection) =>
 					!connection.dependency ||
-					!(connection.dependency instanceof HarmonyImportDependency)
+					(!Dependency.canConcatenate(connection.dependency) &&
+						// exports self-references of a concatenated CommonJS module
+						// are rewritten in place, so they don't block concatenation
+						!(
+							originModule === module &&
+							connection.dependency instanceof CommonJsSelfReferenceDependency
+						))
 			);
 			if (selected.length > 0) {
 				nonHarmonyConnections.set(originModule, connections);
@@ -754,6 +903,7 @@ class ModuleConcatenationPlugin {
 		}
 		if (nonHarmonyConnections.size > 0) {
 			/**
+			 * Returns problem description.
 			 * @param {RequestShortener} requestShortener request shortener
 			 * @returns {string} problem description
 			 */
@@ -780,9 +930,13 @@ class ModuleConcatenationPlugin {
 					", "
 				)}`;
 			};
-			statistics.incorrectModuleDependency++;
-			failureCache.set(module, problem); // cache failures for performance
-			return problem;
+			return cacheFailure(
+				failureCache,
+				statistics,
+				"incorrectModuleDependency",
+				module,
+				problem
+			);
 		}
 
 		if (runtime !== undefined && typeof runtime !== "string") {
@@ -815,6 +969,7 @@ class ModuleConcatenationPlugin {
 			}
 			if (otherRuntimeConnections.length > 0) {
 				/**
+				 * Returns problem description.
 				 * @param {RequestShortener} requestShortener request shortener
 				 * @returns {string} problem description
 				 */
@@ -832,9 +987,13 @@ class ModuleConcatenationPlugin {
 								/** @type {RuntimeSpec} */ (runtimeCondition)
 							)})`
 					).join(", ")}`;
-				statistics.incorrectRuntimeCondition++;
-				failureCache.set(module, problem); // cache failures for performance
-				return problem;
+				return cacheFailure(
+					failureCache,
+					statistics,
+					"incorrectRuntimeCondition",
+					module,
+					problem
+				);
 			}
 		}
 
@@ -860,15 +1019,20 @@ class ModuleConcatenationPlugin {
 				possibleModules,
 				candidates,
 				failureCache,
+				moduleRuntimeCache,
 				chunkGraph,
 				false,
 				statistics
 			);
 			if (problem) {
 				if (backup !== undefined) config.rollback(backup);
-				statistics.importerFailed++;
-				failureCache.set(module, problem); // cache failures for performance
-				return problem;
+				return cacheFailure(
+					failureCache,
+					statistics,
+					"importerFailed",
+					module,
+					problem
+				);
 			}
 		}
 
@@ -885,14 +1049,18 @@ class ModuleConcatenationPlugin {
 
 class ConcatConfiguration {
 	/**
+	 * Creates an instance of ConcatConfiguration.
 	 * @param {Module} rootModule the root module
 	 * @param {RuntimeSpec} runtime the runtime
+	 * @param {Chunk[]} rootChunks the chunks the root module is in
 	 */
-	constructor(rootModule, runtime) {
+	constructor(rootModule, runtime, rootChunks) {
 		/** @type {Module} */
 		this.rootModule = rootModule;
 		/** @type {RuntimeSpec} */
 		this.runtime = runtime;
+		/** @type {Chunk[]} the root's chunks, reused for every candidate's chunk check */
+		this.rootChunks = rootChunks;
 		/** @type {Set<Module>} */
 		this.modules = new Set();
 		this.modules.add(rootModule);
@@ -901,6 +1069,7 @@ class ConcatConfiguration {
 	}
 
 	/**
+	 * Processes the provided module.
 	 * @param {Module} module the module
 	 */
 	add(module) {
@@ -908,6 +1077,7 @@ class ConcatConfiguration {
 	}
 
 	/**
+	 * Returns true, when the module is in the module set.
 	 * @param {Module} module the module
 	 * @returns {boolean} true, when the module is in the module set
 	 */
@@ -920,6 +1090,7 @@ class ConcatConfiguration {
 	}
 
 	/**
+	 * Adds the provided module to the concat configuration.
 	 * @param {Module} module the module
 	 * @param {Problem} problem the problem
 	 */
@@ -928,6 +1099,7 @@ class ConcatConfiguration {
 	}
 
 	/**
+	 * Gets warnings sorted.
 	 * @returns {Warnings} warnings
 	 */
 	getWarningsSorted() {
@@ -943,6 +1115,7 @@ class ConcatConfiguration {
 	}
 
 	/**
+	 * Returns modules as set.
 	 * @returns {Set<Module>} modules as set
 	 */
 	getModules() {
@@ -954,6 +1127,7 @@ class ConcatConfiguration {
 	}
 
 	/**
+	 * Processes the provided snapshot.
 	 * @param {number} snapshot snapshot
 	 */
 	rollback(snapshot) {