@ui5/builder 3.0.7 → 3.0.9

package/lib/processors/themeBuilderWorker.js

@@ -0,0 +1,278 @@
+import workerpool from "workerpool";
+import themeBuilder from "./themeBuilder.js";
+import {createResource} from "@ui5/fs/resourceFactory";
+import {Buffer} from "node:buffer";
+
+/**
+ * Task to build library themes.
+ *
+ * @private
+ * @function default
+ * @static
+ *
+ * @param {object} parameters Parameters
+ * @param {MessagePort} parameters.fsInterfacePort
+ * @param {object[]} parameters.themeResources Input array of Uint8Array transferable objects
+ * 	that are the less sources to build upon. By nature those are @ui5/fs/Resource.
+ * @param {object} parameters.options Less compiler options
+ * @returns {Promise<object[]>} Resulting array of Uint8Array transferable objects
+ */
+export default async function execThemeBuild({
+	fsInterfacePort,
+	themeResources = [],
+	options = {}
+}) {
+	const fsThemeResources = deserializeResources(themeResources);
+	const fsReader = new FsWorkerThreadInterface(fsInterfacePort);
+
+	const result = await themeBuilder({
+		resources: fsThemeResources,
+		fs: fsReader,
+		options
+	});
+
+	return serializeResources(result);
+}
+
+/**
+ * Casts @ui5/fs/Resource-s into an Uint8Array transferable object
+ *
+ * @param {@ui5/fs/Resource[]} resourceCollection
+ * @returns {Promise<object[]>}
+ */
+export async function serializeResources(resourceCollection) {
+	return Promise.all(
+		resourceCollection.map(async (res) => ({
+			buffer: await res.getBuffer(),
+			path: res.getPath()
+		}))
+	);
+}
+
+/**
+ * Casts Uint8Array into @ui5/fs/Resource-s transferable object
+ *
+ * @param {Promise<object[]>} resources
+ * @returns {@ui5/fs/Resource[]}
+ */
+export function deserializeResources(resources) {
+	return resources.map((res) => {
+		// res.buffer is an Uint8Array object and needs to be cast
+		// to a Buffer in order to be read correctly.
+		return createResource({path: res.path, buffer: Buffer.from(res.buffer)});
+	});
+}
+
+/**
+ * "@ui5/fs/fsInterface" like class that uses internally
+ * "@ui5/fs/fsInterface", implements its methods, and
+ * sends the results to a MessagePort.
+ *
+ * Used in the main thread in a combination with FsWorkerThreadInterface.
+ */
+export class FsMainThreadInterface {
+	#comPorts = new Set();
+	#fsInterfaceReader = null;
+	#cache = Object.create(null);
+
+	/**
+	 * Constructor
+	 *
+	 * @param {@ui5/fs/fsInterface} fsInterfaceReader Reader for the Resources
+	 */
+	constructor(fsInterfaceReader) {
+		if (!fsInterfaceReader) {
+			throw new Error("fsInterfaceReader is mandatory argument");
+		}
+
+		this.#fsInterfaceReader = fsInterfaceReader;
+	}
+
+	/**
+	 * Adds MessagePort and starts listening for requests on it.
+	 *
+	 * @param {MessagePort} comPort port1 from a {code}MessageChannel{/code}
+	 */
+	startCommunication(comPort) {
+		if (!comPort) {
+			throw new Error("Communication channel is mandatory argument");
+		}
+
+		this.#comPorts.add(comPort);
+		comPort.on("message", (e) => this.#onMessage(e, comPort));
+		comPort.on("close", () => comPort.close());
+	}
+
+	/**
+	 * Ends MessagePort communication.
+	 *
+	 * @param {MessagePort} comPort port1 to remove from handling.
+	 */
+	endCommunication(comPort) {
+		comPort.close();
+		this.#comPorts.delete(comPort);
+	}
+
+	/**
+	 * Destroys the FsMainThreadInterface
+	 */
+	cleanup() {
+		this.#comPorts.forEach((comPort) => comPort.close());
+		this.#cache = null;
+		this.#fsInterfaceReader = null;
+	}
+
+	/**
+	 * Handles messages from the MessagePort
+	 *
+	 * @param {object} e data to construct the request
+	 * @param {string} e.action Action to perform. Corresponds to the names of
+	 * 	the public methods of "@ui5/fs/fsInterface"
+	 * @param {string} e.fsPath Path of the Resource
+	 * @param {object} e.options Options for "readFile" action
+	 * @param {MessagePort} comPort The communication channel
+	 */
+	#onMessage(e, comPort) {
+		switch (e.action) {
+		case "readFile":
+			this.#doRequest(comPort, {action: "readFile", fsPath: e.fsPath, options: e.options});
+			break;
+		case "stat":
+			this.#doRequest(comPort, {action: "stat", fsPath: e.fsPath});
+			break;
+		}
+	}
+
+	/**
+	 * Requests a Resource from the "@ui5/fs/fsInterface" and sends it to the worker threads
+	 * via postMessage.
+	 *
+	 * @param {MessagePort} comPort The communication channel
+	 * @param {object} parameters
+	 * @param {string} parameters.action Action to perform. Corresponds to the names of
+	 * 	the public methods of "@ui5/fs/fsInterface" and triggers this method of the
+	 *	"@ui5/fs/fsInterface" instance.
+	 * @param {string} parameters.fsPath Path of the Resource
+	 * @param {object} parameters.options Options for "readFile" action
+	 */
+	async #doRequest(comPort, {action, fsPath, options}) {
+		const cacheKey = `${fsPath}-${action}`;
+		if (!this.#cache[cacheKey]) {
+			this.#cache[cacheKey] = new Promise((res) => {
+				if (action === "readFile") {
+					this.#fsInterfaceReader.readFile(fsPath, options, (error, result) => res({error, result}));
+				} else if (action === "stat") {
+					this.#fsInterfaceReader.stat(fsPath, (error, result) =>
+						// The Stat object has some special methods that sometimes cannot be serialized
+						// properly in the postMessage. In this scenario, we do not need those methods,
+						// but just to check whether stats has resolved to something.
+						res(JSON.parse(JSON.stringify({error, result})))
+					);
+				} else {
+					res({error: new Error(`Action "${action}" is not available.`), result: null});
+				}
+			});
+		}
+
+		const fromCache = await this.#cache[cacheKey];
+		comPort.postMessage({action, fsPath, ...fromCache});
+	}
+}
+
+/**
+ * "@ui5/fs/fsInterface" like class that uses internally
+ * "@ui5/fs/fsInterface", implements its methods, and
+ * requests resources via MessagePort.
+ *
+ * Used in the main thread in a combination with FsMainThreadInterface.
+ */
+export class FsWorkerThreadInterface {
+	#comPort = null;
+	#callbacks = [];
+	#cache = Object.create(null);
+
+	/**
+	 * Constructor
+	 *
+	 * @param {MessagePort} comPort Communication port
+	 */
+	constructor(comPort) {
+		if (!comPort) {
+			throw new Error("Communication port is mandatory argument");
+		}
+
+		this.#comPort = comPort;
+		comPort.on("message", this.#onMessage.bind(this));
+		comPort.on("close", this.#onClose.bind(this));
+	}
+
+	/**
+	 * Handles messages from MessagePort
+	 *
+	 * @param {object} e
+	 * @param {string} e.action Action to perform. Corresponds to the names of
+	 * 	the public methods of "@ui5/fs/fsInterface"
+	 * @param {string} e.fsPath Path of the Resource
+	 * @param {*} e.result Response from the "action".
+	 * @param {object} e.error Error from the "action".
+	 */
+	#onMessage(e) {
+		const cbObject = this.#callbacks.find((cb) => cb.action === e.action && cb.fsPath === e.fsPath);
+
+		if (cbObject) {
+			this.#cache[`${e.fsPath}-${e.action}`] = {error: e.error, result: e.result};
+			this.#callbacks.splice(this.#callbacks.indexOf(cbObject), 1);
+			cbObject.callback(e.error, e.result);
+		} else {
+			throw new Error("No callback found for this message! Possible hang for the thread!", e);
+		}
+	}
+
+	/**
+	 * End communication
+	 */
+	#onClose() {
+		this.#comPort.close();
+		this.#cache = null;
+	}
+
+	/**
+	 * Makes a request via the MessagePort
+	 *
+	 * @param {object} parameters
+	 * @param {string} parameters.action Action to perform. Corresponds to the names of
+	 * 	the public methods.
+	 * @param {string} parameters.fsPath Path of the Resource
+	 * @param {object} parameters.options Options for "readFile" action
+	 * @param {Function} callback Callback to call when the "action" is executed and ready.
+	 */
+	#doRequest({action, fsPath, options}, callback) {
+		const cacheKey = `${fsPath}-${action}`;
+
+		if (this.#cache[cacheKey]) {
+			const {result, error} = this.#cache[cacheKey];
+			callback(error, result);
+		} else {
+			this.#callbacks.push({action, fsPath, callback});
+			this.#comPort.postMessage({action, fsPath, options});
+		}
+	}
+
+	readFile(fsPath, options, callback) {
+		this.#doRequest({action: "readFile", fsPath, options}, callback);
+	}
+
+	stat(fsPath, callback) {
+		this.#doRequest({action: "stat", fsPath}, callback);
+	}
+}
+
+// Test execution via ava is never done on the main thread
+/* istanbul ignore else */
+if (!workerpool.isMainThread) {
+	// Script got loaded through workerpool
+	// => Create a worker and register public functions
+	workerpool.worker({
+		execThemeBuild
+	});
+}

package/lib/tasks/buildThemes.js

@@ -1,9 +1,44 @@
 import path from "node:path";
-import themeBuilder from "../processors/themeBuilder.js";
 import fsInterface from "@ui5/fs/fsInterface";
 import ReaderCollectionPrioritized from "@ui5/fs/ReaderCollectionPrioritized";
 import {getLogger} from "@ui5/logger";
 const log = getLogger("builder:tasks:buildThemes");
+import {fileURLToPath} from "node:url";
+import os from "node:os";
+import workerpool from "workerpool";
+import {deserializeResources, serializeResources, FsMainThreadInterface} from "../processors/themeBuilderWorker.js";
+
+let pool;
+
+function getPool(taskUtil) {
+	if (!pool) {
+		const MIN_WORKERS = 2;
+		const MAX_WORKERS = 4;
+		const osCpus = os.cpus().length || 1;
+		const maxWorkers = Math.max(Math.min(osCpus - 1, MAX_WORKERS), MIN_WORKERS);
+
+		log.verbose(`Creating workerpool with up to ${maxWorkers} workers (available CPU cores: ${osCpus})`);
+		const workerPath = fileURLToPath(new URL("../processors/themeBuilderWorker.js", import.meta.url));
+		pool = workerpool.pool(workerPath, {
+			workerType: "thread",
+			maxWorkers
+		});
+		taskUtil.registerCleanupTask(() => {
+			log.verbose(`Terminating workerpool`);
+			const poolToBeTerminated = pool;
+			pool = null;
+			poolToBeTerminated.terminate();
+		});
+	}
+	return pool;
+}
+
+async function buildThemeInWorker(taskUtil, options, transferList) {
+	const toTransfer = transferList ? {transfer: transferList} : undefined;
+
+	return getPool(taskUtil).exec("execThemeBuild", [options], toTransfer);
+}
+
 
 /**
  * @public
@@ -19,6 +54,8 @@ const log = getLogger("builder:tasks:buildThemes");
  * @param {object} parameters Parameters
  * @param {@ui5/fs/DuplexCollection} parameters.workspace DuplexCollection to read and write files
  * @param {@ui5/fs/AbstractReader} parameters.dependencies Reader or Collection to read dependency files
+ * @param {@ui5/builder/tasks/TaskUtil|object} [parameters.taskUtil] TaskUtil instance.
+ *    Required to run buildThemes in parallel execution mode.
  * @param {object} parameters.options Options
  * @param {string} parameters.options.projectName Project name
  * @param {string} parameters.options.inputPattern Search pattern for *.less files to be built
@@ -29,9 +66,10 @@ const log = getLogger("builder:tasks:buildThemes");
  * @returns {Promise<undefined>} Promise resolving with <code>undefined</code> once data has been written
  */
 export default async function({
-	workspace, dependencies,
+	workspace, dependencies, taskUtil,
 	options: {
-		projectName, inputPattern, librariesPattern, themesPattern, compress, cssVariables
+		projectName, inputPattern, librariesPattern, themesPattern, compress,
+		cssVariables
 	}
 }) {
 	const combo = new ReaderCollectionPrioritized({
@@ -41,7 +79,7 @@ export default async function({
 
 	compress = compress === undefined ? true : compress;
 
-	const pAllResources = workspace.byGlob(inputPattern);
+	const pThemeResources = workspace.byGlob(inputPattern);
 	let pAvailableLibraries;
 	let pAvailableThemes;
 	if (librariesPattern) {
@@ -78,7 +116,7 @@ export default async function({
 			});
 	}
 
-	let allResources = await pAllResources;
+	let themeResources = await pThemeResources;
 
 	const isAvailable = function(resource) {
 		let libraryAvailable = false;
@@ -128,17 +166,48 @@ export default async function({
 				log.verbose(`Available sap.ui.core themes: ${availableThemes.join(", ")}`);
 			}
 		}
-		allResources = allResources.filter(isAvailable);
+		themeResources = themeResources.filter(isAvailable);
 	}
 
-	const processedResources = await themeBuilder({
-		resources: allResources,
-		fs: fsInterface(combo),
-		options: {
-			compress,
-			cssVariables: !!cssVariables
-		}
-	});
+	let processedResources;
+	const useWorkers = !!taskUtil;
+	if (useWorkers) {
+		const threadMessageHandler = new FsMainThreadInterface(fsInterface(combo));
+
+		processedResources = await Promise.all(themeResources.map(async (themeRes) => {
+			const {port1, port2} = new MessageChannel();
+			threadMessageHandler.startCommunication(port1);
+
+			const result = await buildThemeInWorker(taskUtil, {
+				fsInterfacePort: port2,
+				themeResources: await serializeResources([themeRes]),
+				options: {
+					compress,
+					cssVariables: !!cssVariables,
+				},
+			}, [port2]);
+
+			threadMessageHandler.endCommunication(port1);
+
+			return result;
+		}))
+			.then((resources) => Array.prototype.concat.apply([], resources))
+			.then(deserializeResources);
+
+		threadMessageHandler.cleanup();
+	} else {
+		// Do not use workerpool
+		const themeBuilder = (await import("../processors/themeBuilder.js")).default;
+
+		processedResources = await themeBuilder({
+			resources: themeResources,
+			fs: fsInterface(combo),
+			options: {
+				compress,
+				cssVariables: !!cssVariables,
+			}
+		});
+	}
 
 	await Promise.all(processedResources.map((resource) => {
 		return workspace.write(resource);

package/lib/tasks/jsdoc/generateJsdoc.js

@@ -154,12 +154,7 @@ const utils = {
 			virBasePath: "/resources/"
 		});
 
-		let allResources;
-		if (workspace.byGlobSource) { // API only available on duplex collections
-			allResources = await workspace.byGlobSource(pattern);
-		} else {
-			allResources = await workspace.byGlob(pattern);
-		}
+		const allResources = await workspace.byGlob(pattern);
 
 		// write all resources to the tmp folder
 		await Promise.all(allResources.map((resource) => fsTarget.write(resource)));

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@ui5/builder",
-	"version": "3.0.7",
+	"version": "3.0.9",
 	"description": "UI5 Tooling - Builder",
 	"author": {
 		"name": "SAP SE",
@@ -38,7 +38,7 @@
 		"unit": "rimraf test/tmp && ava",
 		"unit-verbose": "rimraf test/tmp && cross-env UI5_LOG_LVL=verbose ava --verbose --serial",
 		"unit-watch": "rimraf test/tmp && ava --watch",
-		"unit-xunit": "rimraf test/tmp && ava --no-worker-threads --node-arguments=\"--experimental-loader=@istanbuljs/esm-loader-hook\" --tap --timeout=1m | tap-xunit --dontUseCommentsAsTestNames=true > test-results.xml",
+		"unit-xunit": "rimraf test/tmp && ava --node-arguments=\"--experimental-loader=@istanbuljs/esm-loader-hook\" --tap --timeout=1m | tap-xunit --dontUseCommentsAsTestNames=true > test-results.xml",
 		"unit-inspect": "cross-env UI5_LOG_LVL=verbose ava debug --break",
 		"coverage": "rimraf test/tmp && nyc ava --node-arguments=\"--experimental-loader=@istanbuljs/esm-loader-hook\"",
 		"coverage-xunit": "nyc --reporter=text --reporter=text-summary --reporter=cobertura npm run unit-xunit",
@@ -69,7 +69,8 @@
 		"nodeArguments": [
 			"--loader=esmock",
 			"--no-warnings"
-		]
+		],
+		"workerThreads": false
 	},
 	"nyc": {
 		"reporter": [
@@ -123,20 +124,20 @@
 		"cheerio": "1.0.0-rc.12",
 		"escape-unicode": "^0.2.0",
 		"escope": "^4.0.0",
-		"espree": "^9.6.0",
+		"espree": "^9.6.1",
 		"graceful-fs": "^4.2.11",
 		"jsdoc": "^4.0.2",
 		"less-openui5": "^0.11.6",
 		"pretty-data": "^0.40.0",
 		"rimraf": "^5.0.1",
 		"semver": "^7.5.4",
-		"terser": "^5.18.2",
+		"terser": "^5.19.2",
 		"workerpool": "^6.4.0",
 		"xml2js": "^0.6.0"
 	},
 	"devDependencies": {
 		"@istanbuljs/esm-loader-hook": "^0.2.0",
-		"@ui5/project": "^3.4.1",
+		"@ui5/project": "^3.4.2",
 		"ava": "^5.3.1",
 		"chai": "^4.3.7",
 		"chai-fs": "^2.0.0",
@@ -144,11 +145,11 @@
 		"cross-env": "^7.0.3",
 		"depcheck": "^1.4.3",
 		"docdash": "^2.0.1",
-		"eslint": "^8.44.0",
+		"eslint": "^8.45.0",
 		"eslint-config-google": "^0.14.0",
 		"eslint-plugin-ava": "^14.0.0",
-		"eslint-plugin-jsdoc": "^46.4.3",
-		"esmock": "^2.3.1",
+		"eslint-plugin-jsdoc": "^46.4.4",
+		"esmock": "^2.3.2",
 		"nyc": "^15.1.0",
 		"open-cli": "^7.2.0",
 		"recursive-readdir": "^2.2.3",