5etools-utils 0.1.24 → 0.2.0

package/lib/TestJson.js

@@ -1,11 +1,17 @@
+import * as os from "os";
 import * as path from "path";
+import * as url from "url";
+import {Worker} from "worker_threads";
 
 import Ajv2020 from "ajv/dist/2020.js";
-import addFormats from "ajv-formats"
+import addFormats from "ajv-formats";
 import * as jsonSourceMap from "json-source-map";
 
 import * as uf from "./UtilFs.js";
 import * as um from "./UtilMisc.js";
+import {WorkerList, Deferred} from "./WorkerList.js";
+
+const __dirname = url.fileURLToPath(new URL(".", import.meta.url));
 
 const _IS_SORT_RESULTS = !process.env.VET_TEST_JSON_RESULTS_UNSORTED;
 const _IS_TRIM_RESULTS = !process.env.VET_TEST_JSON_RESULTS_UNTRIMMED;
@@ -65,8 +71,10 @@ class JsonTester {
 
 	_getFileErrors ({filePath}) { return `${filePath}\n${JSON.stringify(this._ajv.errors, null, 2)}`; }
 
-	_handleError ({filePath, errors, errorsFull, data}) {
-		if (!process.env.CI) errorsFull.push(this._getFileErrors({filePath}));
+	_getErrors ({filePath, data}) {
+		const out = {errors: [], errorsFull: []};
+
+		if (!process.env.CI) out.errorsFull.push(this._getFileErrors({filePath}));
 
 		// Sort the deepest errors to the bottom, as these are the ones we're most likely to be the ones we care about
 		//   manually checking.
@@ -91,7 +99,9 @@ class JsonTester {
 
 		const error = this._getFileErrors({filePath});
 		um.error(this._tagLog, error);
-		errors.push(error);
+		out.errors.push(error);
+
+		return out;
 	}
 
 	_doLoadSchema () {
@@ -111,18 +121,26 @@ class JsonTester {
 		this._isSchemaLoaded = true;
 	}
 
-	_doRunOnDir ({isFailFast, dir, errors, errorsFull, ...opts} = {}) {
-		for (const filePath of uf.listJsonFiles(dir, opts)) {
-			um.info(this._tagLog, `\tValidating "${filePath}"...`);
+	getFileErrors ({filePath} = {}) {
+		this._doLoadSchema();
 
-			const data = uf.readJSON(filePath);
-			this._addImplicits(data);
-			const valid = this._ajv.validate(this._fnGetSchemaId(filePath), data);
+		um.info(this._tagLog, `\tValidating "${filePath}"...`);
 
-			if (!valid) {
-				this._handleError({filePath, errors, errorsFull, data});
-				if (isFailFast) break;
-			}
+		const data = uf.readJSON(filePath);
+		this._addImplicits(data);
+
+		const isValid = this._ajv.validate(this._fnGetSchemaId(filePath), data);
+		if (isValid) return {};
+
+		return this._getErrors({filePath, data});
+	}
+
+	_doRunOnDir ({isFailFast, dir, errors, errorsFull, ...opts} = {}) {
+		for (const filePath of uf.listJsonFiles(dir, opts)) {
+			const {errors: errorsFile = [], errorsFull: errorsFullFile = []} = this.getFileErrors({filePath});
+			errors.push(...errorsFile);
+			errorsFull.push(...errorsFullFile);
+			if (isFailFast && (errorsFile.length || errorsFullFile.length)) break;
 		}
 	}
 
@@ -134,8 +152,6 @@ class JsonTester {
 	getErrors (dir, opts = {}) {
 		um.info(this._tagLog, `Validating JSON against schema`);
 
-		this._doLoadSchema();
-
 		const errors = [];
 		const errorsFull = [];
 
@@ -147,8 +163,6 @@ class JsonTester {
 	getErrorsOnDirs ({isFailFast = false} = {}) {
 		um.info(this._tagLog, `Validating JSON against schema`);
 
-		this._doLoadSchema();
-
 		const errors = [];
 		const errorsFull = [];
 
@@ -159,6 +173,81 @@ class JsonTester {
 
 		return {errors, errorsFull};
 	}
+
+	async pGetErrorsOnDirsWorkers ({isFailFast = false} = {}) {
+		um.info(this._tagLog, `Validating JSON against schema`);
+
+		const cntWorkers = Math.max(1, os.cpus().length - 1);
+
+		const errors = [];
+		const errorsFull = [];
+
+		const fileQueue = [];
+
+		uf.runOnDirs((dir) => {
+			fileQueue.push(...uf.listJsonFiles(dir));
+		});
+
+		const workerList = new WorkerList();
+
+		const workers = [...new Array(cntWorkers)]
+			.map(() => {
+				// Relative `Worker` paths do not function in packages, so give an exact path
+				const worker = new Worker(path.join(__dirname, "TestJsonWorker.js"));
+
+				worker.on("message", (msg) => {
+					switch (msg.type) {
+						case "ready":
+						case "done": {
+							if (msg.payload.isError) {
+								errors.push(...msg.payload.errors);
+								errorsFull.push(...msg.payload.errorsFull);
+
+								if (isFailFast) workers.forEach(worker => worker.postMessage({type: "cancel"}));
+							}
+
+							if (worker.dIsActive) worker.dIsActive.resolve();
+							workerList.add(worker);
+
+							break;
+						}
+					}
+				});
+
+				worker.on("error", e => console.error(e));
+
+				worker.postMessage({
+					type: "init",
+					payload: {
+						dirSchema: this._dirSchema,
+						tagLog: this._tagLog,
+						strFnGetSchemaId: this._fnGetSchemaId.toString(),
+					},
+				});
+
+				return worker;
+			});
+
+		while (fileQueue.length) {
+			if (isFailFast && errors.length) break;
+
+			const file = fileQueue.shift();
+			const worker = await workerList.get();
+
+			worker.dIsActive = new Deferred();
+			worker.postMessage({
+				type: "work",
+				payload: {
+					file,
+				},
+			});
+		}
+
+		await Promise.all(workers.map(it => it.dIsActive.promise));
+		await Promise.all(workers.map(it => it.terminate()));
+
+		return {errors, errorsFull};
+	}
 }
 
 export {JsonTester, LOG_TAG};

package/lib/TestJsonWorker.js

@@ -0,0 +1,57 @@
+import {isMainThread, parentPort} from "worker_threads";
+import {JsonTester} from "./TestJson.js";
+
+if (isMainThread) throw new Error(`Worker must not be started in main thread!`);
+
+let jsonTester;
+let isCancelled = false;
+
+parentPort
+	.on("message", msg => {
+		switch (msg.type) {
+			case "init": {
+				const {dirSchema, tagLog, strFnGetSchemaId} = msg.payload;
+
+				jsonTester = new JsonTester({
+					dirSchema,
+					tagLog,
+					fnGetSchemaId: eval(strFnGetSchemaId), // eslint-disable-line no-eval
+				});
+
+				parentPort.postMessage({
+					type: "ready",
+					payload: {},
+				});
+
+				break;
+			}
+
+			case "cancel": {
+				isCancelled = true;
+				break;
+			}
+
+			case "work": {
+				if (isCancelled) {
+					parentPort.postMessage({
+						type: "done",
+						payload: {},
+					});
+					return;
+				}
+
+				const {errors = [], errorsFull = []} = jsonTester.getFileErrors({filePath: msg.payload.file});
+
+				parentPort.postMessage({
+					type: "done",
+					payload: {
+						isError: !!(errors.length || errorsFull.length),
+						errors,
+						errorsFull,
+					},
+				});
+
+				break;
+			}
+		}
+	});

package/lib/WorkerList.js

@@ -0,0 +1,47 @@
+class Deferred {
+	constructor () {
+		this._resolve = null;
+		this._reject = null;
+		this._promise = new Promise((resolve, reject) => {
+			this._resolve = resolve;
+			this._reject = reject;
+		});
+	}
+
+	get resolve () { return this._resolve; }
+	get reject () { return this._reject; }
+	get promise () { return this._promise; }
+}
+
+class WorkerList {
+	/**
+	 * @param {Array} workers
+	 */
+	constructor ({workers = null} = {}) {
+		this._workers = workers || [];
+		this._deferredQueue = [];
+	}
+
+	add (worker) {
+		this._workers.push(worker);
+
+		while (this._deferredQueue.length && this._workers.length) {
+			const d = this._deferredQueue.shift();
+			d.resolve(this._workers.shift());
+		}
+	}
+
+	get () {
+		if (this._workers.length) {
+			return Promise.resolve(this._workers.shift());
+		}
+
+		const d = new Deferred();
+		this._deferredQueue.push(d);
+		return d.promise;
+	}
+
+	get cntAvailableWorkers () { return this._workers.length; }
+}
+
+export {Deferred, WorkerList};

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "5etools-utils",
-	"version": "0.1.24",
+	"version": "0.2.0",
 	"description": "Shared utilities for the 5etools ecosystem.",
 	"type": "module",
 	"main": "lib/Api.js",
@@ -11,9 +11,9 @@
 		"lib": "./lib"
 	},
 	"scripts": {
-		"test": "npm run test:js",
-		"test:js": "eslint lib",
-		"lint:js": "eslint lib --fix"
+		"test": "npm run lint:js && npm run test:js",
+		"test:js": "node --experimental-vm-modules node_modules/jest/bin/jest.js",
+		"lint:js": "eslint lib test --fix"
 	},
 	"repository": {
 		"type": "git",
@@ -27,10 +27,11 @@
 	"homepage": "https://github.com/TheGiddyLimit/5etools-utils#readme",
 	"dependencies": {
 		"ajv": "^8.11.2",
-		"json-source-map": "^0.6.1",
-		"ajv-formats": "^2.1.1"
+		"ajv-formats": "^2.1.1",
+		"json-source-map": "^0.6.1"
 	},
 	"devDependencies": {
-		"eslint": "^8.11.0"
+		"eslint": "^8.11.0",
+		"jest": "^29.3.1"
 	}
 }