5etools-utils 0.10.30 → 0.11.0

package/lib/BrewTester.js

@@ -20,6 +20,7 @@ class _BrewTesterJson {
 		const opts = program.opts();
 
 		const jsonTester = new JsonTester({mode, tagLog: this._LOG_TAG, fnGetSchemaId: () => "homebrew.json"});
+		await jsonTester.pInit();
 
 		let results;
 		if (program.args[0]) {
@@ -32,7 +33,7 @@ class _BrewTesterJson {
 			results = await jsonTester.pGetErrorsOnDirsWorkers({isFailFast: !this._IS_FAIL_SLOW});
 		}
 
-		const {errors, errorsFull} = results;
+		const {errors, errorsFull, isUnknownError = false} = results;
 
 		if (errors.length) {
 			if (!process.env.CI) {
@@ -45,6 +46,11 @@ class _BrewTesterJson {
 			process.exit(1);
 		}
 
+		if (isUnknownError) {
+			console.error(`Unknown error when testing! (See above logs)`);
+			process.exit(1);
+		}
+
 		if (!errors.length) Um.info(this._LOG_TAG, `Schema test passed.`);
 	}
 }

package/lib/TestJson.js

@@ -11,6 +11,8 @@ import NP from "number-precision";
 import * as Uf from "./UtilFs.js";
 import Um from "./UtilMisc.js";
 import {WorkerList, Deferred} from "./WorkerList.js";
+import {ObjectWalker} from "./ObjectWalker.js";
+import {UrlUtil} from "./UrlUtil.js";
 
 const __dirname = url.fileURLToPath(new URL(".", import.meta.url));
 
@@ -87,7 +89,11 @@ class JsonTester {
 		});
 		// endregion
 
-		this._isSchemaLoaded = false;
+		this._pLoadSchema = null;
+	}
+
+	async pInit () {
+		await this._pDoLoadSchemas();
 	}
 
 	_getFileErrors ({filePath}) { return `${filePath}\n${JSON.stringify(this._ajv.errors, null, 2)}`; }
@@ -141,30 +147,55 @@ class JsonTester {
 		);
 	}
 
-	_doLoadSchemas () {
-		if (this._isSchemaLoaded) return;
+	async _pDoLoadSchemas () {
+		await (this._pLoadSchema ||= (async () => {
+			const remoteUrls = new Set();
 
-		Uf.listJsonFiles(this._dirSchema)
-			.forEach(filePath => {
-				filePath = path.normalize(filePath);
+			Uf.listJsonFiles(this._dirSchema)
+				.forEach(filePath => {
+					filePath = path.normalize(filePath);
 
-				const relativeFilePath = path.relative(this._dirSchema, filePath)
-					.replace(/\\/g, "/");
+					const relativeFilePath = path.relative(this._dirSchema, filePath)
+						.replace(/\\/g, "/");
 
-				this._ajv.addSchema(Uf.readJsonSync(filePath), relativeFilePath);
-			});
+					const json = Uf.readJsonSync(filePath);
+					this._ajv.addSchema(json, relativeFilePath);
+
+					ObjectWalker.walk({
+						obj: json,
+						filePath,
+						primitiveHandlers: {
+							object: obj => {
+								if (!obj.$ref) return;
+
+								if (!UrlUtil.isUrl(obj.$ref, {protocols: ["http:", "https:"]})) return;
+
+								const url = new URL(obj.$ref);
+								url.hash = "";
+								remoteUrls.add(String(url));
+							},
+						},
+					});
+				});
 
-		this._isSchemaLoaded = true;
+			if (!remoteUrls.size) return;
+
+			await Promise.all(
+				Array.from(remoteUrls).map(async remoteUrl => {
+					this._ajv.addSchema(await (await fetch(remoteUrl)).json(), remoteUrl);
+				}),
+			);
+
+			return true;
+		})());
 	}
 
-	_hasSchema (schemaId) {
-		this._doLoadSchemas();
+	async _pHasSchema (schemaId) {
+		await this._pDoLoadSchemas();
 		return !!this._ajv.schemas[schemaId];
 	}
 
 	getFileErrors ({filePath} = {}) {
-		this._doLoadSchemas();
-
 		Um.info(this._tagLog, `\tValidating "${filePath}"...`);
 
 		const data = Uf.readJsonSync(filePath);
@@ -231,16 +262,19 @@ class JsonTester {
 		}
 
 		// region Verify that every file path maps to a valid schema
-		fileQueue.forEach(filePath => {
-			const schemaId = this._fnGetSchemaId(filePath);
-			if (!schemaId) throw new Error(`Failed to get schema ID for file path "${filePath}"`);
-			if (!this._hasSchema(schemaId)) throw new Error(`No schema loaded with schema ID "${schemaId}"`);
-			return schemaId;
-		});
+		await Promise.all(
+			fileQueue.map(async filePath => {
+				const schemaId = this._fnGetSchemaId(filePath);
+				if (!schemaId) throw new Error(`Failed to get schema ID for file path "${filePath}"`);
+				if (!(await this._pHasSchema(schemaId))) throw new Error(`No schema loaded with schema ID "${schemaId}"`);
+				return schemaId;
+			}),
+		);
 		// endregion
 
 		const workerList = new WorkerList();
 
+		let cntFailures = 0;
 		const workers = [...new Array(cntWorkers)]
 			.map(() => {
 				// Relative `Worker` paths do not function in packages, so give an exact path
@@ -265,7 +299,10 @@ class JsonTester {
 					}
 				});
 
-				worker.on("error", e => console.error(e));
+				worker.on("error", e => {
+					console.error(e);
+					cntFailures++;
+				});
 
 				worker.postMessage({
 					type: "init",
@@ -297,7 +334,7 @@ class JsonTester {
 		await Promise.all(workers.map(it => it.dIsActive?.promise));
 		await Promise.all(workers.map(it => it.terminate()));
 
-		return {errors, errorsFull};
+		return {errors, errorsFull, isUnknownError: !!cntFailures};
 	}
 }
 

package/lib/TestJsonWorker.js

@@ -7,16 +7,17 @@ let jsonTester;
 let isCancelled = false;
 
 parentPort
-	.on("message", msg => {
+	.on("message", async msg => {
 		switch (msg.type) {
 			case "init": {
 				const {dirSchema, tagLog, strFnGetSchemaId} = msg.payload;
 
-				jsonTester = new JsonTester({
+				jsonTester ||= new JsonTester({
 					dirSchema,
 					tagLog,
 					fnGetSchemaId: eval(strFnGetSchemaId), // eslint-disable-line no-eval
 				});
+				await jsonTester.pInit();
 
 				parentPort.postMessage({
 					type: "ready",

package/lib/UrlUtil.js

@@ -0,0 +1,19 @@
+export class UrlUtil {
+	/**
+	 * @param str
+	 * @param {?Array<string>} protocols
+	 * @return {*|boolean}
+	 */
+	static isUrl (str, {protocols = null} = {}) {
+		let url;
+
+		try {
+			url = new URL(str);
+		} catch (e) {
+			return false;
+		}
+
+		if (protocols == null) return true;
+		return protocols.includes(url.protocol);
+	}
+}

package/lib/UtilFs.js

@@ -1,5 +1,6 @@
 import * as fs from "fs";
 import * as path from "path";
+import {getCleanString} from "./UtilClean.js";
 
 function isDirectory (path) {
 	return fs.lstatSync(path).isDirectory();
@@ -29,8 +30,10 @@ function readJsonSync (path, {isIncludeRaw = false} = {}) {
 	}
 }
 
-function writeJsonSync (filePath, data) {
-	return fs.writeFileSync(filePath, `${JSON.stringify(data, null, "\t")}\n`, "utf-8");
+function writeJsonSync (filePath, data, {isClean = false} = {}) {
+	let str = `${JSON.stringify(data, null, "\t")}\n`;
+	if (isClean) str = getCleanString(str);
+	return fs.writeFileSync(filePath, str, "utf-8");
 }
 
 /**

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "5etools-utils",
-	"version": "0.10.30",
+	"version": "0.11.0",
 	"description": "Shared utilities for the 5etools ecosystem.",
 	"type": "module",
 	"main": "lib/Api.js",