@ui5/builder 3.0.9 → 3.1.0

package/CHANGELOG.md

@@ -2,10 +2,23 @@
 All notable changes to this project will be documented in this file.  
 This project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html).
 
-A list of unreleased changes can be found [here](https://github.com/SAP/ui5-builder/compare/v3.0.9...HEAD).
+A list of unreleased changes can be found [here](https://github.com/SAP/ui5-builder/compare/v3.1.0...HEAD).
+
+<a name="v3.1.0"></a>
+## [v3.1.0] - 2023-10-11
+### Bug Fixes
+- **bundle/Builder:** Add missing 'names' attribute to generated source maps [`57e0e50`](https://github.com/SAP/ui5-builder/commit/57e0e5047638a9a704a696b8af7780fbbceefbc4)
+
+### Features
+- Validate apiVersion property in Library.init ([#943](https://github.com/SAP/ui5-builder/issues/943)) [`52bf258`](https://github.com/SAP/ui5-builder/commit/52bf25842a59c3fa1ddbe71b482b232e18b55288)
+- **Minifier:** Support input source maps ([#780](https://github.com/SAP/ui5-builder/issues/780)) [`67ecb27`](https://github.com/SAP/ui5-builder/commit/67ecb27e44a2d84e6b2203f31049220dcbcd41f0)
+
+### Reverts
+- [INTERNAL] Azure Pipelines: Switch back to Node 20.5.x
+
 
 <a name="v3.0.9"></a>
-## [v3.0.9] - 2023-07-25
+## [v3.0.9] - 2023-07-26
 
 <a name="v3.0.8"></a>
 ## [v3.0.8] - 2023-07-19
@@ -819,6 +832,7 @@ to load the custom bundle file instead.
 
 ### Features
 - Add ability to configure component preloads and custom bundles [`2241e5f`](https://github.com/SAP/ui5-builder/commit/2241e5ff98fd95f1f80cc74959655ae7a9c660e7)
+[v3.1.0]: https://github.com/SAP/ui5-builder/compare/v3.0.9...v3.1.0
 [v3.0.9]: https://github.com/SAP/ui5-builder/compare/v3.0.8...v3.0.9
 [v3.0.8]: https://github.com/SAP/ui5-builder/compare/v3.0.7...v3.0.8
 [v3.0.7]: https://github.com/SAP/ui5-builder/compare/v3.0.6...v3.0.7

package/lib/lbt/analyzer/analyzeLibraryJS.js

@@ -61,6 +61,9 @@ async function analyze(resource) {
 						libInfo.elements = getStringArray(value, true);
 					} else if ( ["designtime", "dependencies", "extensions", "name", "version"].includes(key) ) {
 						// do nothing, for all other supported properties
+					} else if ( key === "apiVersion" &&
+						(value.type === Syntax.Literal && typeof value.value === "number") ) {
+						// just a validation
 					} else {
 						log.error(
 							"Unexpected property '" + key +

package/lib/lbt/bundle/Builder.js

@@ -18,7 +18,7 @@ import BundleWriter from "./BundleWriter.js";
 import {getLogger} from "@ui5/logger";
 const log = getLogger("lbt:bundle:Builder");
 
-const sourceMappingUrlPattern = /\/\/# sourceMappingURL=(.+)\s*$/;
+const sourceMappingUrlPattern = /\/\/# sourceMappingURL=(\S+)\s*$/;
 const httpPattern = /^https?:\/\//i;
 const xmlHtmlPrePattern = /<(?:\w+:)?pre\b/;
 
@@ -332,11 +332,18 @@ class BundleBuilder {
 			throw new Error("No source map provided");
 		}
 
+		// Reminder on the structure of line-segments in the map:
+		// [generatedCodeColumn, sourceIndex, sourceCodeLine, sourceCodeColumn, nameIndex]
 		if (map.mappings.startsWith(";")) {
 			// If first line is not already mapped (typical for comments or parentheses), add a mapping to
 			// make sure that dev-tools (especially Chrome's) don't choose the end of the preceding module
 			// when the user tries to set a breakpoint from the bundle file
 			map.mappings = "AAAA" + map.mappings;
+		} else if (this.outW.columnOffset === 0 && !map.mappings.startsWith("A")) {
+			// If first column of the first line is not already mapped, add a mapping for the same reason as above.
+			// This is typical for transpiled code, where there is a bunch of generated code at the beginning that
+			// can't be mapped to the original source
+			map.mappings = "AAAA," + map.mappings;
 		}
 
 		map.sourceRoot = path.posix.relative(
@@ -592,9 +599,7 @@ class BundleBuilder {
 				`Attempting to find a source map resource based on the module's path: ${sourceMapFileCandidate}`);
 			try {
 				const sourceMapResource = await this.pool.findResource(sourceMapFileCandidate);
-				if (sourceMapResource) {
-					moduleSourceMap = (await sourceMapResource.buffer()).toString();
-				}
+				moduleSourceMap = (await sourceMapResource.buffer()).toString();
 			} catch (e) {
 				// No input source map
 				log.silly(`Could not find a source map for module ${moduleName}: ${e.message}`);
@@ -772,6 +777,7 @@ async function transform(changes, moduleContent, moduleSourceMap) {
 function createTransientSourceMap({moduleName, moduleContent, includeContent = false}) {
 	const sourceMap = {
 		version: 3,
+		names: [],
 		sources: [moduleName],
 		// TODO: check whether moduleContent.match() with \n is better w.r.t performance/memory usage
 		mappings: encodeMappings(moduleContent.split("\n").map((line, i) => {

package/lib/processors/minifier.js

@@ -1,5 +1,6 @@
 import {fileURLToPath} from "node:url";
 import posixPath from "node:path/posix";
+import {promisify} from "node:util";
 import os from "node:os";
 import workerpool from "workerpool";
 import Resource from "@ui5/fs/Resource";
@@ -13,6 +14,9 @@ const MAX_WORKERS = 4;
 const osCpus = os.cpus().length || 1;
 const maxWorkers = Math.max(Math.min(osCpus - 1, MAX_WORKERS), MIN_WORKERS);
 
+const sourceMappingUrlPattern = /\/\/# sourceMappingURL=(\S+)\s*$/;
+const httpPattern = /^https?:\/\//i;
+
 // Shared workerpool across all executions until the taskUtil cleanup is triggered
 let pool;
 
@@ -38,6 +42,63 @@ async function minifyInWorker(options, taskUtil) {
 	return getPool(taskUtil).exec("execMinification", [options]);
 }
 
+async function extractAndRemoveSourceMappingUrl(resource) {
+	const resourceContent = await resource.getString();
+	const resourcePath = resource.getPath();
+	const sourceMappingUrlMatch = resourceContent.match(sourceMappingUrlPattern);
+	if (sourceMappingUrlMatch) {
+		const sourceMappingUrl = sourceMappingUrlMatch[1];
+		if (log.isLevelEnabled("silly")) {
+			log.silly(`Found source map reference in content of resource ${resourcePath}: ${sourceMappingUrl}`);
+		}
+
+		// Strip sourceMappingURL from the resource to be minified
+		// It is not required anymore and will be replaced for in the minified resource
+		// and its debug variant anyways
+		resource.setString(resourceContent.replace(sourceMappingUrlPattern, ""));
+		return sourceMappingUrl;
+	}
+	return null;
+}
+
+async function getSourceMapFromUrl({sourceMappingUrl, resourcePath, readFile}) {
+	// =======================================================================
+	// This code is almost identical to code located in lbt/bundle/Builder.js
+	// Please try to update both places when making changes
+	// =======================================================================
+	if (sourceMappingUrl.startsWith("data:")) {
+		// Data-URI indicates an inline source map
+		const expectedTypeAndEncoding = "data:application/json;charset=utf-8;base64,";
+		if (sourceMappingUrl.startsWith(expectedTypeAndEncoding)) {
+			const base64Content = sourceMappingUrl.slice(expectedTypeAndEncoding.length);
+			// Create a resource with a path suggesting it's the source map for the resource
+			// (which it is but inlined)
+			return Buffer.from(base64Content, "base64").toString();
+		} else {
+			log.warn(
+				`Source map reference in resource ${resourcePath} is a data URI but has an unexpected` +
+				`encoding: ${sourceMappingUrl}. Expected it to start with ` +
+				`"data:application/json;charset=utf-8;base64,"`);
+		}
+	} else if (httpPattern.test(sourceMappingUrl)) {
+		log.warn(`Source map reference in resource ${resourcePath} is an absolute URL. ` +
+			`Currently, only relative URLs are supported.`);
+	} else if (posixPath.isAbsolute(sourceMappingUrl)) {
+		log.warn(`Source map reference in resource ${resourcePath} is an absolute path. ` +
+			`Currently, only relative paths are supported.`);
+	} else {
+		const sourceMapPath = posixPath.join(posixPath.dirname(resourcePath), sourceMappingUrl);
+
+		try {
+			const sourceMapContent = await readFile(sourceMapPath);
+			return sourceMapContent.toString();
+		} catch (e) {
+			// No input source map
+			log.warn(`Unable to read source map for resource ${resourcePath}: ${e.message}`);
+		}
+	}
+}
+
 /**
  * @public
  * @module @ui5/builder/processors/minifier
@@ -62,9 +123,16 @@ async function minifyInWorker(options, taskUtil) {
  *
  * @param {object} parameters Parameters
  * @param {@ui5/fs/Resource[]} parameters.resources List of resources to be processed
+ * @param {fs|module:@ui5/fs/fsInterface} parameters.fs Node fs or custom
+ *    [fs interface]{@link module:@ui5/fs/fsInterface}. Required when setting "readSourceMappingUrl" to true
  * @param {@ui5/builder/tasks/TaskUtil|object} [parameters.taskUtil] TaskUtil instance.
  *    Required when using the <code>useWorkers</code> option
  * @param {object} [parameters.options] Options
+ * @param {boolean} [parameters.options.readSourceMappingUrl=false]
+ *   Whether to make use of any existing source maps referenced in the resources to be minified. Use this option to
+ *   preserve references to the original source files, such as TypeScript files, in the generated source map.<br>
+ *   If a resource has been modified by a previous task, any existing source map will be ignored regardless of this
+ *    setting. This is to ensure that no inconsistent source maps are used. Check the verbose log for details.
  * @param {boolean} [parameters.options.addSourceMappingUrl=true]
  *   Whether to add a sourceMappingURL reference to the end of the minified resource
  * @param {boolean} [parameters.options.useWorkers=false]
@@ -72,8 +140,14 @@ async function minifyInWorker(options, taskUtil) {
  * @returns {Promise<module:@ui5/builder/processors/minifier~MinifierResult[]>}
  *   Promise resolving with object of resource, dbgResource and sourceMap
  */
-export default async function({resources, taskUtil, options: {addSourceMappingUrl = true, useWorkers = false} = {}}) {
+export default async function({
+	resources, fs, taskUtil, options: {readSourceMappingUrl = false, addSourceMappingUrl = true, useWorkers = false
+	} = {}}) {
 	let minify;
+	if (readSourceMappingUrl && !fs) {
+		throw new Error(`Option 'readSourceMappingUrl' requires parameter 'fs' to be provided`);
+	}
+
 	if (useWorkers) {
 		if (!taskUtil) {
 			// TaskUtil is required for worker support
@@ -86,12 +160,11 @@ export default async function({resources, taskUtil, options: {addSourceMappingUr
 	}
 
 	return Promise.all(resources.map(async (resource) => {
-		const dbgPath = resource.getPath().replace(debugFileRegex, "-dbg$1");
-		const dbgResource = await resource.clone();
-		dbgResource.setPath(dbgPath);
+		const resourcePath = resource.getPath();
+		const dbgPath = resourcePath.replace(debugFileRegex, "-dbg$1");
+		const dbgFilename = posixPath.basename(dbgPath);
 
 		const filename = posixPath.basename(resource.getPath());
-		const code = await resource.getString();
 
 		const sourceMapOptions = {
 			filename
@@ -99,7 +172,81 @@ export default async function({resources, taskUtil, options: {addSourceMappingUr
 		if (addSourceMappingUrl) {
 			sourceMapOptions.url = filename + ".map";
 		}
-		const dbgFilename = posixPath.basename(dbgPath);
+
+		// Remember contentModified flag before making changes to the resource via setString
+		const resourceContentModified = resource.getSourceMetadata()?.contentModified;
+
+		// In any case: Extract *and remove* source map reference from resource before cloning it
+		const sourceMappingUrl = await extractAndRemoveSourceMappingUrl(resource);
+
+		const code = await resource.getString();
+		// Create debug variant based off the original resource before minification
+		const dbgResource = await resource.clone();
+		dbgResource.setPath(dbgPath);
+
+		let dbgSourceMapResource;
+		if (sourceMappingUrl) {
+			if (resourceContentModified) {
+				log.verbose(
+					`Source map found in resource will be ignored because the resource has been ` +
+					`modified in a previous task: ${resourcePath}`);
+			} else if (readSourceMappingUrl) {
+				// Try to find a source map reference in the to-be-minified resource
+				// If we find one, provide it to terser as an input source map and keep using it for the
+				// debug variant of the resource
+				const sourceMapContent = await getSourceMapFromUrl({
+					sourceMappingUrl,
+					resourcePath,
+					readFile: promisify(fs.readFile)
+				});
+
+				if (sourceMapContent) {
+					const sourceMapJson = JSON.parse(sourceMapContent);
+
+					if (sourceMapJson.sections) {
+						// TODO 4.0
+						// Module "@jridgewell/trace-mapping" (used by Terser) can't handle index map sections lacking
+						// a "names" array. Since this is a common occurrence for UI5 Tooling bundles, we search for
+						// such cases here and fix them until https://github.com/jridgewell/trace-mapping/pull/29 is
+						// resolved and Terser upgraded the dependency
+
+						// Create a dedicated clone before modifying the source map as to not alter the debug source map
+						const clonedSourceMapJson = JSON.parse(sourceMapContent);
+						clonedSourceMapJson.sections.forEach(({map}) => {
+							if (!map.names) {
+								// Add missing names array
+								map.names = [];
+							}
+						});
+						// Use modified source map as input source map
+						sourceMapOptions.content = JSON.stringify(clonedSourceMapJson);
+					} else {
+						// Provide source map to terser as "input source map"
+						sourceMapOptions.content = sourceMapContent;
+					}
+
+					// Use the original source map for the debug variant of the resource
+					// First update the file reference within the source map
+					sourceMapJson.file = dbgFilename;
+
+					// Then create a new resource
+					dbgSourceMapResource = new Resource({
+						string: JSON.stringify(sourceMapJson),
+						path: dbgPath + ".map"
+					});
+					// And reference the resource in the debug resource
+					dbgResource.setString(code + `//# sourceMappingURL=${dbgFilename}.map\n`);
+				}
+			} else {
+				// If the original resource content was unmodified and the input source map was not parsed,
+				// re-add the original source map reference to the debug variant
+				if (!sourceMappingUrl.startsWith("data:") && !sourceMappingUrl.endsWith(filename + ".map")) {
+					// Do not re-add inline source maps as well as references to the source map of
+					// the minified resource
+					dbgResource.setString(code + `//# sourceMappingURL=${sourceMappingUrl}\n`);
+				}
+			}
+		}
 
 		const result = await minify({
 			filename,
@@ -112,6 +259,9 @@ export default async function({resources, taskUtil, options: {addSourceMappingUr
 			path: resource.getPath() + ".map",
 			string: result.map
 		});
-		return {resource, dbgResource, sourceMapResource};
+		return {resource, dbgResource, sourceMapResource, dbgSourceMapResource};
 	}));
 }
+
+export const __localFunctions__ = (process.env.NODE_ENV === "test") ?
+	{getSourceMapFromUrl} : undefined;

package/lib/processors/minifierWorker.js

@@ -62,7 +62,9 @@ export default async function execMinification({
 		// Note: err.filename contains the debug-name
 		throw new Error(
 			`Minification failed with error: ${err.message} in file ${filename} ` +
-			`(line ${err.line}, col ${err.col}, pos ${err.pos})`);
+			`(line ${err.line}, col ${err.col}, pos ${err.pos})`, {
+				cause: err
+			});
 	}
 }
 

package/lib/tasks/bundlers/utils/createModuleNameMapping.js

@@ -18,8 +18,8 @@ export default function({resources, taskUtil}) {
 	const moduleNameMapping = Object.create(null);
 	for (let i = resources.length - 1; i >= 0; i--) {
 		const resource = resources[i];
-		if (taskUtil.getTag(resource, taskUtil.STANDARD_TAGS.IsDebugVariant)) {
-			const resourcePath = resource.getPath();
+		const resourcePath = resource.getPath();
+		if (resourcePath.endsWith(".js") && taskUtil.getTag(resource, taskUtil.STANDARD_TAGS.IsDebugVariant)) {
 			const nonDbgPath = getNonDebugName(resourcePath);
 			if (!nonDbgPath) {
 				throw new Error(`Failed to resolve non-debug name for ${resourcePath}`);

package/lib/tasks/minify.js

@@ -1,4 +1,5 @@
 import minifier from "../processors/minifier.js";
+import fsInterface from "@ui5/fs/fsInterface";
 
 /**
  * @public
@@ -19,20 +20,29 @@ import minifier from "../processors/minifier.js";
  * @param {string} parameters.options.pattern Pattern to locate the files to be processed
  * @param {boolean} [parameters.options.omitSourceMapResources=false] Whether source map resources shall
  * 		be tagged as "OmitFromBuildResult" and no sourceMappingURL shall be added to the minified resource
+ * @param {boolean} [parameters.options.useInputSourceMaps=true] Whether to make use of any existing source
+ * 		maps referenced in the resources to be minified. Use this option to preserve reference to the original
+ * 		source files, such as TypeScript files, in the generated source map.
  * @returns {Promise<undefined>} Promise resolving with <code>undefined</code> once data has been written
  */
-export default async function({workspace, taskUtil, options: {pattern, omitSourceMapResources = false}}) {
+export default async function({
+	workspace, taskUtil, options: {pattern, omitSourceMapResources = false, useInputSourceMaps = true
+	}}) {
 	const resources = await workspace.byGlob(pattern);
 	const processedResources = await minifier({
 		resources,
+		fs: fsInterface(workspace),
 		taskUtil,
 		options: {
 			addSourceMappingUrl: !omitSourceMapResources,
+			readSourceMappingUrl: !!useInputSourceMaps,
 			useWorkers: !!taskUtil,
 		}
 	});
 
-	return Promise.all(processedResources.map(async ({resource, dbgResource, sourceMapResource}) => {
+	return Promise.all(processedResources.map(async ({
+		resource, dbgResource, sourceMapResource, dbgSourceMapResource
+	}) => {
 		if (taskUtil) {
 			taskUtil.setTag(resource, taskUtil.STANDARD_TAGS.HasDebugVariant);
 			taskUtil.setTag(dbgResource, taskUtil.STANDARD_TAGS.IsDebugVariant);
@@ -40,11 +50,18 @@ export default async function({workspace, taskUtil, options: {pattern, omitSourc
 			if (omitSourceMapResources) {
 				taskUtil.setTag(sourceMapResource, taskUtil.STANDARD_TAGS.OmitFromBuildResult);
 			}
+			if (dbgSourceMapResource) {
+				taskUtil.setTag(dbgSourceMapResource, taskUtil.STANDARD_TAGS.IsDebugVariant);
+				if (omitSourceMapResources) {
+					taskUtil.setTag(dbgSourceMapResource, taskUtil.STANDARD_TAGS.OmitFromBuildResult);
+				}
+			}
 		}
 		return Promise.all([
 			workspace.write(resource),
 			workspace.write(dbgResource),
-			workspace.write(sourceMapResource)
+			workspace.write(sourceMapResource),
+			dbgSourceMapResource && workspace.write(dbgSourceMapResource)
 		]);
 	}));
 }

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@ui5/builder",
-	"version": "3.0.9",
+	"version": "3.1.0",
 	"description": "UI5 Tooling - Builder",
 	"author": {
 		"name": "SAP SE",
@@ -119,7 +119,7 @@
 	},
 	"dependencies": {
 		"@jridgewell/sourcemap-codec": "^1.4.15",
-		"@ui5/fs": "^3.0.4",
+		"@ui5/fs": "^3.0.5",
 		"@ui5/logger": "^3.0.0",
 		"cheerio": "1.0.0-rc.12",
 		"escape-unicode": "^0.2.0",
@@ -129,31 +129,33 @@
 		"jsdoc": "^4.0.2",
 		"less-openui5": "^0.11.6",
 		"pretty-data": "^0.40.0",
-		"rimraf": "^5.0.1",
+		"rimraf": "^5.0.5",
 		"semver": "^7.5.4",
-		"terser": "^5.19.2",
-		"workerpool": "^6.4.0",
-		"xml2js": "^0.6.0"
+		"terser": "^5.21.0",
+		"workerpool": "^6.5.0",
+		"xml2js": "^0.6.2"
 	},
 	"devDependencies": {
 		"@istanbuljs/esm-loader-hook": "^0.2.0",
-		"@ui5/project": "^3.4.2",
+		"@jridgewell/trace-mapping": "^0.3.19",
+		"@ui5/project": "^3.7.1",
 		"ava": "^5.3.1",
-		"chai": "^4.3.7",
+		"chai": "^4.3.10",
 		"chai-fs": "^2.0.0",
 		"chokidar-cli": "^3.0.0",
 		"cross-env": "^7.0.3",
-		"depcheck": "^1.4.3",
-		"docdash": "^2.0.1",
-		"eslint": "^8.45.0",
+		"depcheck": "^1.4.6",
+		"docdash": "^2.0.2",
+		"eslint": "^8.51.0",
 		"eslint-config-google": "^0.14.0",
 		"eslint-plugin-ava": "^14.0.0",
-		"eslint-plugin-jsdoc": "^46.4.4",
-		"esmock": "^2.3.2",
+		"eslint-plugin-jsdoc": "^46.8.2",
+		"esmock": "^2.5.2",
+		"line-column": "^1.0.2",
 		"nyc": "^15.1.0",
 		"open-cli": "^7.2.0",
 		"recursive-readdir": "^2.2.3",
-		"sinon": "^15.2.0",
+		"sinon": "^16.1.0",
 		"tap-xunit": "^2.4.1"
 	}
 }