5etools-utils 0.15.3 → 0.15.4

package/README.md

@@ -3,6 +3,11 @@
 Available commands:
 
 ```
+npx clean-html
+npx test-file-names [--name-regex <regex>]
+npx test-file-contents --img-repo-name <repo> --url-prefix-expected <prefix>
 npx test-json-brew [file] [--dir <dir>]
 npx test-json-ua [file] [--dir <dir>]
 ```
+
+Programmatic: `BrewTester` includes helpers such as `pTestImgDirectories({dirAllowlist, pathImgDir})`.

package/bin/clean-html.js

@@ -0,0 +1,5 @@
+#!/usr/bin/env node
+
+import {BrewCleanerHtml} from "../lib/BrewCleanerHtml.js";
+
+await BrewCleanerHtml.pRun();

package/bin/test-file-contents.js

@@ -0,0 +1,19 @@
+#!/usr/bin/env node
+
+import {Command} from "commander";
+import {BrewTester} from "../lib/BrewTester.js";
+
+const program = new Command()
+	.requiredOption("--img-repo-name <repo>", "Image repo short name for logs, e.g. \"homebrew-img\"")
+	.requiredOption("--url-prefix-expected <prefix>", "Expected URL prefix to match against image URLs")
+	.option("--path-error-log <path>", "Path to write full check output", "./_test/test-data.error.log")
+;
+
+program.parse(process.argv);
+const opts = program.opts();
+
+await BrewTester.pTestFileContents({
+	imgRepoName: opts.imgRepoName,
+	urlPrefixExpected: opts.urlPrefixExpected,
+	pathErrorLog: opts.pathErrorLog,
+});

package/bin/test-file-names.js

@@ -1,5 +1,15 @@
 #!/usr/bin/env node
 
 import {BrewTester} from "../lib/BrewTester.js";
+import {Command} from "commander";
 
-await BrewTester.pTestFileNames();
+const program = new Command()
+	.option("--name-regex <regex>", "Regex pattern used to validate file names")
+;
+
+program.parse(process.argv);
+const opts = program.opts();
+
+await BrewTester.pTestFileNames({
+	reNameFormat: opts.nameRegex ? new RegExp(opts.nameRegex) : null,
+});

package/lib/Api.js

@@ -3,6 +3,7 @@ import Um from "./UtilMisc.js";
 import * as Uf from "./UtilFs.js";
 import {BrewIndexGenerator} from "./BrewIndexGenerator.js";
 import {BrewCleaner} from "./BrewCleaner.js";
+import {BrewCleanerHtml} from "./BrewCleanerHtml.js";
 import {BrewTimestamper} from "./BrewTimestamper.js";
 import {BrewTester} from "./BrewTester.js";
 import {getCleanJson, getCleanString} from "./UtilClean.js";
@@ -16,6 +17,7 @@ export {
 	Uf,
 	BrewIndexGenerator,
 	BrewCleaner,
+	BrewCleanerHtml,
 	BrewTimestamper,
 	BrewTester,
 	getCleanJson,

package/lib/BrewCleanerHtml.js

@@ -0,0 +1,175 @@
+import {Worker} from "node:worker_threads";
+import fs from "node:fs";
+import os from "node:os";
+import he from "he";
+import sanitizeHtml from "sanitize-html";
+import {getCleanJson} from "./UtilClean.js";
+import {ObjectWalker} from "./ObjectWalker.js";
+import * as Uf from "./UtilFs.js";
+import Um from "./UtilMisc.js";
+import {Deferred, WorkerList} from "./WorkerList.js";
+
+export class BrewCleanerHtml {
+	static _LOG_TAG = `HTML`;
+
+	static _OPTS_SANITIZE = {
+		allowedTags: [
+			// region Custom things which look like tags
+			"<$name$>",
+			// endregion
+		],
+		allowedAttributes: {},
+	};
+
+	static _getCleanFileMeta ({file}) {
+		const fileData = Uf.readJsonSync(file);
+
+		const messages = [];
+
+		const {_meta, _test} = fileData;
+		delete fileData._meta;
+		delete fileData._test;
+
+		const keyStack = [];
+		const objectStack = [];
+
+		const isInFoundryDescriptionEffect = () => {
+			if (objectStack.at(-1)?.key !== "system.description.value") return false;
+			return keyStack.at(-1) === "changes" && ["effects", "foundryEffects"].includes(keyStack.at(-2));
+		};
+
+		const fileOut = ObjectWalker.walk({
+			obj: fileData,
+			filePath: file,
+			primitiveHandlers: {
+				string: (str, {lastKey}) => {
+					if (lastKey === "value" && isInFoundryDescriptionEffect()) return str;
+
+					const clean = he.unescape(
+						sanitizeHtml(
+							str,
+							this._OPTS_SANITIZE,
+						),
+					);
+
+					if (clean !== str) {
+						const msg = `Sanitized ${keyStack.map(k => `"${k}"`).join(" -> ")}:\n${str}\n${clean}`;
+						messages.push(msg);
+						Um.info(this._LOG_TAG, msg);
+					}
+
+					return clean;
+				},
+				preObject: (obj) => objectStack.push(obj),
+				postObject: () => objectStack.pop(),
+				preArray: (_, {lastKey}) => keyStack.push(lastKey),
+				postArray: () => keyStack.pop(),
+			},
+			isModify: true,
+		});
+
+		const out = {$schema: fileOut.$schema, _meta, _test};
+		Object.assign(out, fileOut);
+
+		return {
+			messages,
+			out,
+		};
+	}
+
+	static async _pUpdateDir (dir) {
+		Uf.listJsonFiles(dir)
+			.forEach(file => {
+				const {messages, out} = this._getCleanFileMeta({file});
+				if (!messages?.length) return;
+
+				fs.writeFileSync(file, getCleanJson(out));
+			});
+	}
+
+	static async pRun () {
+		await Uf.pRunOnDirs(
+			async (dir) => {
+				Um.info(this._LOG_TAG, `Sanitizing HTML in dir "${dir}"...`);
+				await this._pUpdateDir(dir);
+			},
+			{
+				isSerial: true,
+			},
+		);
+		Um.info(this._LOG_TAG, "Done!");
+	}
+
+	static getFileMessages ({file}) {
+		return this._getCleanFileMeta({file});
+	}
+
+	static async pGetErrorsOnDirsWorkers ({isFailFast = false} = {}) {
+		Um.info(this._LOG_TAG, `Testing for HTML...`);
+
+		const cntWorkers = Math.max(1, os.cpus().length - 1);
+
+		const messages = [];
+
+		const fileQueue = [];
+		Uf.runOnDirs((dir) => fileQueue.push(...Uf.listJsonFiles(dir)));
+
+		const workerList = new WorkerList();
+
+		let cntFailures = 0;
+		const workers = [...new Array(cntWorkers)]
+			.map(() => {
+				const worker = new Worker(new URL("./BrewCleanerHtmlWorker.js", import.meta.url));
+
+				worker.on("message", (msg) => {
+					switch (msg.type) {
+						case "ready":
+						case "done": {
+							if (msg.payload.isError) {
+								messages.push(...msg.payload.messages);
+
+								if (isFailFast) workers.forEach(worker => worker.postMessage({type: "cancel"}));
+							}
+
+							if (worker.dIsActive) worker.dIsActive.resolve();
+							workerList.add(worker);
+
+							break;
+						}
+					}
+				});
+
+				worker.on("error", e => {
+					console.error(e);
+					cntFailures++;
+				});
+
+				worker.postMessage({
+					type: "init",
+					payload: {},
+				});
+
+				return worker;
+			});
+
+		while (fileQueue.length) {
+			if (isFailFast && messages.length) break;
+
+			const file = fileQueue.shift();
+			const worker = await workerList.get();
+
+			worker.dIsActive = new Deferred();
+			worker.postMessage({
+				type: "work",
+				payload: {
+					file,
+				},
+			});
+		}
+
+		await Promise.all(workers.map(it => it.dIsActive?.promise));
+		await Promise.all(workers.map(it => it.terminate()));
+
+		return {messages, isUnknownError: !!cntFailures};
+	}
+}

package/lib/BrewCleanerHtmlWorker.js

@@ -0,0 +1,47 @@
+import {isMainThread, parentPort} from "node:worker_threads";
+import {BrewCleanerHtml} from "./BrewCleanerHtml.js";
+
+if (isMainThread) throw new Error(`Worker must not be started in main thread!`);
+
+let isCancelled = false;
+
+parentPort
+	.on("message", async msg => {
+		switch (msg.type) {
+			case "init": {
+				parentPort.postMessage({
+					type: "ready",
+					payload: {},
+				});
+
+				break;
+			}
+
+			case "cancel": {
+				isCancelled = true;
+				break;
+			}
+
+			case "work": {
+				if (isCancelled) {
+					parentPort.postMessage({
+						type: "done",
+						payload: {},
+					});
+					return;
+				}
+
+				const {messages = []} = BrewCleanerHtml.getFileMessages({file: msg.payload.file});
+
+				parentPort.postMessage({
+					type: "done",
+					payload: {
+						isError: !!messages.length,
+						messages,
+					},
+				});
+
+				break;
+			}
+		}
+	});

package/lib/BrewTester/BrewTesterFileContents.js

@@ -0,0 +1,207 @@
+import fs from "node:fs";
+import {BrewTesterBase} from "./BrewTesterBase.js";
+import {DataTester, DataTesterBase, BraceCheck, EscapeCharacterCheck} from "../TestData.js";
+import * as Uf from "../UtilFs.js";
+import {ObjectWalker} from "../ObjectWalker.js";
+import {UtilSource} from "../UtilSource.js";
+import Um from "../UtilMisc.js";
+
+class _CopySourceCheck extends DataTesterBase {
+	static _FileState = class {
+		sources;
+		dependencies;
+		internalCopies;
+
+		constructor ({contents}) {
+			this.sources = new Set(
+				(contents._meta?.sources?.map(src => src?.json) || [])
+					.filter(Boolean),
+			);
+			this.dependencies = Object.fromEntries(
+				Object.entries(contents._meta?.dependencies || {})
+					.map(([prop, arr]) => [prop, new Set(arr)]),
+			);
+			this.internalCopies = new Set(contents._meta?.internalCopies || []);
+		}
+	};
+
+	registerParsedFileCheckers (parsedJsonChecker) {
+		parsedJsonChecker.registerFileHandler(this);
+	}
+
+	handleFile (file, contents) {
+		if (!file.includes("Kobold Press; Scarlet Citadel.json")) return;
+
+		const fileState = new this.constructor._FileState({contents});
+
+		Object.entries(contents)
+			.forEach(([prop, arr]) => {
+				if (prop.startsWith("_")) return;
+				if (!(arr instanceof Array)) return;
+
+				arr.forEach(ent => {
+					const propStack = [prop];
+					const inlineDependencies = new Set();
+
+					ObjectWalker.walk({
+						obj: ent,
+						filePath: file,
+						primitiveHandlers: {
+							preObject: this._onPreObject.bind(this, {propStack, inlineDependencies}),
+							object: this._checkObject.bind(this, {fileState, propStack, inlineDependencies}),
+							postObject: this._onPostObject.bind(this, {propStack, inlineDependencies}),
+						},
+					});
+				});
+			});
+	}
+
+	_onPreObject ({propStack, inlineDependencies}, obj) {
+		if (obj.type !== "statblockInline") return;
+
+		propStack.push(obj.dataType);
+		(obj.dependencies || []).forEach(dep => inlineDependencies.add(dep));
+	}
+
+	_onPostObject ({propStack, inlineDependencies}, obj) {
+		if (obj.type !== "statblockInline") return;
+
+		propStack.pop();
+		inlineDependencies.clear();
+	}
+
+	_checkObject ({fileState, propStack, inlineDependencies}, obj, {filePath}) {
+		if (!obj._copy?.source) return;
+
+		const prop = propStack.at(-1);
+		const sourceCopy = obj._copy.source;
+
+		// Classes/subclasses have an alternate structure.
+		if (["class", "subclass"].includes(prop) && UtilSource.isSiteSource(sourceCopy)) {
+			const classNameLower = obj._copy.className?.toLowerCase();
+			if (
+				fileState.dependencies[prop]?.has(classNameLower)
+				|| inlineDependencies.has(classNameLower)
+			) return;
+		}
+
+		// If a root entity, i.e. not in a `statblockInline`, allow internal copies.
+		if (
+			propStack.length === 1
+			&& fileState.internalCopies.has(prop)
+			&& fileState.sources.has(sourceCopy)
+		) return;
+
+		if (
+			fileState.dependencies[prop]?.has(sourceCopy)
+			|| inlineDependencies.has(sourceCopy)
+		) return;
+
+		this._addMessage(`Entity "${propStack.join(" -> ")}" "${obj.name}" "_copy" source "${sourceCopy}" did not match sources found in dependencies in file "${filePath}"\n`);
+	}
+}
+
+class _ImageUrlCheck extends DataTesterBase {
+	static _RE_IMG_PATH = /^(?<type>img|pdf)\/(?<source>[^/]+)\//;
+
+	static _FileState = class {
+		sources;
+
+		constructor ({contents}) {
+			this.sources = new Set(
+				[
+					...(contents._meta?.sources?.map(src => src?.json) || [])
+						.filter(Boolean)
+						.map(srcJson => srcJson.replace(/:/g, "")),
+					...(contents._test?.additionalImageSources || [])
+						.map(srcJson => srcJson.replace(/:/g, "")),
+				],
+			);
+		}
+	};
+
+	constructor ({imgRepoName, urlPrefixExpected}) {
+		super();
+		this._imgRepoName = imgRepoName;
+		this._urlPrefixExpected = urlPrefixExpected;
+	}
+
+	registerParsedFileCheckers (parsedJsonChecker) {
+		parsedJsonChecker.registerFileHandler(this);
+	}
+
+	handleFile (file, contents) {
+		const fileState = new this.constructor._FileState({contents});
+
+		ObjectWalker.walk({
+			obj: contents,
+			filePath: file,
+			primitiveHandlers: {
+				object: this._checkObject.bind(this, {fileState}),
+			},
+		});
+	}
+
+	_checkObject ({fileState}, obj, {filePath}) {
+		if (obj.type !== "image" || obj.href?.type !== "external" || !obj.href?.url) return;
+
+		const {url} = obj.href;
+		if (!url.toLowerCase().startsWith(this._urlPrefixExpected.toLowerCase())) return;
+
+		const mPath = this.constructor._RE_IMG_PATH.exec(url.slice(this._urlPrefixExpected.length));
+		if (!mPath) {
+			this._addMessage(`Unknown "${this._imgRepoName}" URL pattern in file "${filePath}": "${url}"\n`);
+			return;
+		}
+
+		const {source, type} = mPath.groups;
+		if (fileState.sources.has(source)) return;
+
+		this._addMessage(`Image source part "${source}" in "${this._imgRepoName}" ${type} URL did not match sources found in file "_meta" or "_test" in file "${filePath}": "${url}"\n`);
+	}
+}
+
+export class BrewTesterFileContents extends BrewTesterBase {
+	_LOG_TAG = "FILE_CONTENTS";
+
+	constructor ({imgRepoName, urlPrefixExpected, pathErrorLog} = {}) {
+		super();
+		this._imgRepoName = imgRepoName;
+		this._urlPrefixExpected = urlPrefixExpected;
+		this._pathErrorLog = pathErrorLog || "test-data.error.log";
+	}
+
+	async _pRun () {
+		if (!this._imgRepoName) throw new Error(`Image repo name was required!`);
+		if (!this._urlPrefixExpected) throw new Error(`Expected URL prefix was required!`);
+
+		Um.info(this._LOG_TAG, `Running checks for image repo "${this._imgRepoName}" with URL prefix "${this._urlPrefixExpected}"...`);
+
+		const dataTesters = [
+			new BraceCheck(),
+			new EscapeCharacterCheck(),
+			new _ImageUrlCheck({imgRepoName: this._imgRepoName, urlPrefixExpected: this._urlPrefixExpected}),
+			new _CopySourceCheck(),
+		];
+		DataTester.register({dataTesters});
+
+		await Uf.pRunOnDirs(
+			async (dir) => {
+				Um.info(this._LOG_TAG, `Checking dir "${dir}"...`);
+				await DataTester.pRun(dir, dataTesters);
+			},
+			{
+				isSerial: true,
+			},
+		);
+
+		const outMessage = DataTester.getLogReport(dataTesters);
+		if (!outMessage) {
+			Um.info(this._LOG_TAG, `Complete.`);
+			return;
+		}
+
+		fs.writeFileSync(this._pathErrorLog, outMessage, "utf-8");
+		throw new Error(`Checks failed! See "${this._pathErrorLog}" and logs above.`);
+	}
+}

package/lib/BrewTester/BrewTesterFileNames.js

@@ -8,6 +8,11 @@ export class BrewTesterFileNames extends BrewTesterBase {
 
 	static _RE_NAME_FORMAT = /^[^;]+; .+\.json$/;
 
+	constructor ({reNameFormat = null} = {}) {
+		super();
+		this._reNameFormat = reNameFormat || this.constructor._RE_NAME_FORMAT;
+	}
+
 	async _pRun () {
 		Um.info(this._LOG_TAG, `Testing for incorrect file names...`);
 
@@ -24,8 +29,8 @@ export class BrewTesterFileNames extends BrewTesterBase {
 
 			errors.push(
 				...filenames
-					.filter(it => !this.constructor._RE_NAME_FORMAT.test(it))
-					.map(it => `Filename did not match expected pattern "${this.constructor._RE_NAME_FORMAT.toString()}" extension: ${it}`),
+					.filter(it => !this._reNameFormat.test(it))
+					.map(it => `Filename did not match expected pattern "${this._reNameFormat.toString()}" extension: ${it}`),
 			);
 		});
 

package/lib/BrewTester/BrewTesterImgDirectories.js

@@ -0,0 +1,32 @@
+import fs from "node:fs";
+import {BrewTesterBase} from "./BrewTesterBase.js";
+import Um from "../UtilMisc.js";
+
+export class BrewTesterImgDirectories extends BrewTesterBase {
+	_LOG_TAG = "IMG_DIR";
+
+	constructor ({dirAllowlist = null, pathImgDir = "_img"} = {}) {
+		super();
+		this._dirAllowlist = new Set(dirAllowlist || []);
+		this._pathImgDir = pathImgDir;
+	}
+
+	async _pRun () {
+		if (!this._dirAllowlist.size) throw new Error(`Directory allowlist was required!`);
+
+		if (!fs.existsSync(this._pathImgDir)) {
+			Um.info(this._LOG_TAG, `Directory "${this._pathImgDir}" was not found; skipping.`);
+			return;
+		}
+
+		const extraDirs = fs.readdirSync(this._pathImgDir)
+			.filter(dir => !this._dirAllowlist.has(dir));
+
+		if (!extraDirs.length) {
+			Um.info(this._LOG_TAG, `No unexpected entries found in "${this._pathImgDir}".`);
+			return;
+		}
+
+		throw new Error(`Extra directories found in "${this._pathImgDir}":\n${extraDirs.map(d => `\t${d}`).join("\n")}`);
+	}
+}

package/lib/BrewTester.js

@@ -3,11 +3,15 @@ import {BrewTesterFileLocations} from "./BrewTester/BrewTesterFileLocations.js";
 import {BrewTesterFileNames} from "./BrewTester/BrewTesterFileNames.js";
 import {BrewTesterFileProps} from "./BrewTester/BrewTesterFileProps.js";
 import {BrewTesterEdition} from "./BrewTester/BrewTesterEdition.js";
+import {BrewTesterFileContents} from "./BrewTester/BrewTesterFileContents.js";
+import {BrewTesterImgDirectories} from "./BrewTester/BrewTesterImgDirectories.js";
 
 export class BrewTester {
 	static async pTestJson ({mode, filepath, dir}) { return (new BrewTesterJson({mode, filepath, dir})).pRun(); }
 	static pTestFileLocations () { return (new BrewTesterFileLocations()).pRun(); }
-	static pTestFileNames () { return (new BrewTesterFileNames()).pRun(); }
+	static pTestFileNames ({reNameFormat} = {}) { return (new BrewTesterFileNames({reNameFormat})).pRun(); }
 	static pTestFileProps () { return (new BrewTesterFileProps()).pRun(); }
+	static pTestFileContents ({imgRepoName, urlPrefixExpected, pathErrorLog}) { return (new BrewTesterFileContents({imgRepoName, urlPrefixExpected, pathErrorLog})).pRun(); }
+	static pTestImgDirectories ({dirAllowlist, pathImgDir} = {}) { return (new BrewTesterImgDirectories({dirAllowlist, pathImgDir})).pRun(); }
 	static pTestEditionSources () { return (new BrewTesterEdition()).pRun(); }
 }

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "5etools-utils",
-	"version": "0.15.3",
+	"version": "0.15.4",
 	"description": "Shared utilities for the 5etools ecosystem.",
 	"type": "module",
 	"main": "lib/Api.js",
@@ -12,6 +12,8 @@
 		"lib": "./lib"
 	},
 	"bin": {
+		"clean-html": "bin/clean-html.js",
+		"test-file-contents": "bin/test-file-contents.js",
 		"test-json-brew": "bin/test-json-brew.js",
 		"test-json-ua": "bin/test-json-ua.js",
 		"test-file-names": "bin/test-file-names.js",
@@ -43,8 +45,10 @@
 		"ajv-formats": "^3.0.1",
 		"commander": "^14.0.3",
 		"hasha": "^7.0.0",
+		"he": "^1.2.0",
 		"json-source-map": "^0.6.1",
-		"number-precision": "^1.6.0"
+		"number-precision": "^1.6.0",
+		"sanitize-html": "^2.17.1"
 	},
 	"devDependencies": {
 		"@eslint/js": "^10.0.1",