5etools-utils 0.15.2 → 0.15.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +5 -0
- package/bin/clean-html.js +5 -0
- package/bin/test-edition-sources.js +0 -0
- package/bin/test-file-contents.js +19 -0
- package/bin/test-file-locations.js +0 -0
- package/bin/test-file-names.js +11 -1
- package/bin/test-file-props.js +0 -0
- package/bin/test-json-brew.js +0 -0
- package/bin/test-json-ua.js +0 -0
- package/lib/Api.js +2 -0
- package/lib/BrewCleaner.js +27 -13
- package/lib/BrewCleanerHtml.js +175 -0
- package/lib/BrewCleanerHtmlWorker.js +47 -0
- package/lib/BrewTester/BrewTesterFileContents.js +207 -0
- package/lib/BrewTester/BrewTesterFileNames.js +7 -2
- package/lib/BrewTester/BrewTesterImgDirectories.js +32 -0
- package/lib/BrewTester.js +5 -1
- package/package.json +6 -2
package/README.md
CHANGED
|
@@ -3,6 +3,11 @@
|
|
|
3
3
|
Available commands:
|
|
4
4
|
|
|
5
5
|
```
|
|
6
|
+
npx clean-html
|
|
7
|
+
npx test-file-names [--name-regex <regex>]
|
|
8
|
+
npx test-file-contents --img-repo-name <repo> --url-prefix-expected <prefix>
|
|
6
9
|
npx test-json-brew [file] [--dir <dir>]
|
|
7
10
|
npx test-json-ua [file] [--dir <dir>]
|
|
8
11
|
```
|
|
12
|
+
|
|
13
|
+
Programmatic: `BrewTester` includes helpers such as `pTestImgDirectories({dirAllowlist, pathImgDir})`.
|
|
File without changes
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
|
|
3
|
+
import {Command} from "commander";
|
|
4
|
+
import {BrewTester} from "../lib/BrewTester.js";
|
|
5
|
+
|
|
6
|
+
const program = new Command()
|
|
7
|
+
.requiredOption("--img-repo-name <repo>", "Image repo short name for logs, e.g. \"homebrew-img\"")
|
|
8
|
+
.requiredOption("--url-prefix-expected <prefix>", "Expected URL prefix to match against image URLs")
|
|
9
|
+
.option("--path-error-log <path>", "Path to write full check output", "./_test/test-data.error.log")
|
|
10
|
+
;
|
|
11
|
+
|
|
12
|
+
program.parse(process.argv);
|
|
13
|
+
const opts = program.opts();
|
|
14
|
+
|
|
15
|
+
await BrewTester.pTestFileContents({
|
|
16
|
+
imgRepoName: opts.imgRepoName,
|
|
17
|
+
urlPrefixExpected: opts.urlPrefixExpected,
|
|
18
|
+
pathErrorLog: opts.pathErrorLog,
|
|
19
|
+
});
|
|
File without changes
|
package/bin/test-file-names.js
CHANGED
|
@@ -1,5 +1,15 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
2
|
|
|
3
3
|
import {BrewTester} from "../lib/BrewTester.js";
|
|
4
|
+
import {Command} from "commander";
|
|
4
5
|
|
|
5
|
-
|
|
6
|
+
const program = new Command()
|
|
7
|
+
.option("--name-regex <regex>", "Regex pattern used to validate file names")
|
|
8
|
+
;
|
|
9
|
+
|
|
10
|
+
program.parse(process.argv);
|
|
11
|
+
const opts = program.opts();
|
|
12
|
+
|
|
13
|
+
await BrewTester.pTestFileNames({
|
|
14
|
+
reNameFormat: opts.nameRegex ? new RegExp(opts.nameRegex) : null,
|
|
15
|
+
});
|
package/bin/test-file-props.js
CHANGED
|
File without changes
|
package/bin/test-json-brew.js
CHANGED
|
File without changes
|
package/bin/test-json-ua.js
CHANGED
|
File without changes
|
package/lib/Api.js
CHANGED
|
@@ -3,6 +3,7 @@ import Um from "./UtilMisc.js";
|
|
|
3
3
|
import * as Uf from "./UtilFs.js";
|
|
4
4
|
import {BrewIndexGenerator} from "./BrewIndexGenerator.js";
|
|
5
5
|
import {BrewCleaner} from "./BrewCleaner.js";
|
|
6
|
+
import {BrewCleanerHtml} from "./BrewCleanerHtml.js";
|
|
6
7
|
import {BrewTimestamper} from "./BrewTimestamper.js";
|
|
7
8
|
import {BrewTester} from "./BrewTester.js";
|
|
8
9
|
import {getCleanJson, getCleanString} from "./UtilClean.js";
|
|
@@ -16,6 +17,7 @@ export {
|
|
|
16
17
|
Uf,
|
|
17
18
|
BrewIndexGenerator,
|
|
18
19
|
BrewCleaner,
|
|
20
|
+
BrewCleanerHtml,
|
|
19
21
|
BrewTimestamper,
|
|
20
22
|
BrewTester,
|
|
21
23
|
getCleanJson,
|
package/lib/BrewCleaner.js
CHANGED
|
@@ -131,12 +131,12 @@ export class BrewCleaner {
|
|
|
131
131
|
// - Characters which have significance in a URL
|
|
132
132
|
static _RE_INVALID_CHARS = /[<>:"/\\|?*#%@]/;
|
|
133
133
|
|
|
134
|
-
static _cleanFolder (
|
|
134
|
+
static _cleanFolder ({dir, isDry = false}) {
|
|
135
135
|
const ALL_ERRORS = [];
|
|
136
136
|
|
|
137
|
-
const files = Uf.listJsonFiles(
|
|
137
|
+
const files = Uf.listJsonFiles(dir);
|
|
138
138
|
for (const file of files) {
|
|
139
|
-
|
|
139
|
+
const {raw, json} = Uf.readJsonSync(file, {isIncludeRaw: true});
|
|
140
140
|
|
|
141
141
|
if (this._RE_INVALID_CHARS.test(file.split("/").slice(1).join("/"))) {
|
|
142
142
|
ALL_ERRORS.push(`${file} contained invalid characters!`);
|
|
@@ -148,15 +148,21 @@ export class BrewCleaner {
|
|
|
148
148
|
if (!this._IS_FAIL_SLOW) break;
|
|
149
149
|
}
|
|
150
150
|
|
|
151
|
-
|
|
152
|
-
_BrewFileTester.testFile({ALL_ERRORS, file, contents});
|
|
151
|
+
let jsonOut = _BrewFileCleaner.cleanFile({file, contents: json});
|
|
152
|
+
_BrewFileTester.testFile({ALL_ERRORS, file, contents: jsonOut});
|
|
153
153
|
|
|
154
154
|
if (!this._IS_FAIL_SLOW && ALL_ERRORS.length) break;
|
|
155
155
|
|
|
156
156
|
Um.info(`CLEANER`, `\t- "${file}"...`);
|
|
157
|
-
contents = getCleanJson(contents);
|
|
158
157
|
|
|
159
|
-
|
|
158
|
+
jsonOut = getCleanJson(jsonOut);
|
|
159
|
+
|
|
160
|
+
if (isDry) {
|
|
161
|
+
if (jsonOut !== raw) ALL_ERRORS.push(`${file} would be reformatted!`);
|
|
162
|
+
continue;
|
|
163
|
+
}
|
|
164
|
+
|
|
165
|
+
fs.writeFileSync(file, jsonOut);
|
|
160
166
|
}
|
|
161
167
|
|
|
162
168
|
if (ALL_ERRORS.length) {
|
|
@@ -164,19 +170,27 @@ export class BrewCleaner {
|
|
|
164
170
|
throw new Error(`Errors were found. See above.`);
|
|
165
171
|
}
|
|
166
172
|
|
|
167
|
-
return files.length;
|
|
173
|
+
return {cntFiles: files.length, cntErrors: ALL_ERRORS.length};
|
|
168
174
|
}
|
|
169
175
|
|
|
170
|
-
static run () {
|
|
176
|
+
static run ({isDry = false} = {}) {
|
|
171
177
|
const tStart = Date.now();
|
|
172
178
|
|
|
173
|
-
let
|
|
179
|
+
let ttlFiles = 0;
|
|
180
|
+
let ttlErrors = 0;
|
|
174
181
|
|
|
175
182
|
Uf.runOnDirs((dir) => {
|
|
176
|
-
Um.info(`CLEANER`,
|
|
177
|
-
|
|
183
|
+
Um.info(`CLEANER`, `${isDry ? "Checking" : "Cleaning"} dir "${dir}"...`);
|
|
184
|
+
const {cntFiles, cntErrors} = this._cleanFolder({dir, isDry});
|
|
185
|
+
ttlFiles += cntFiles;
|
|
186
|
+
ttlErrors += cntErrors;
|
|
178
187
|
});
|
|
179
188
|
|
|
180
|
-
Um.info(`CLEANER`,
|
|
189
|
+
Um.info(`CLEANER`, `${isDry ? "Check" : "Cleaning"} complete. ${isDry ? "Checked" : "Cleaned"} ${ttlFiles} file${ttlFiles === 1 ? "" : "s"}. Ran in ${((Date.now() - tStart) / 1000).toFixed(2)}s.`);
|
|
190
|
+
|
|
191
|
+
if (ttlErrors && isDry) {
|
|
192
|
+
Um.info(`CLEANER`, `Found ${ttlErrors} error${ttlErrors === 1 ? "" : "s"}!`);
|
|
193
|
+
process.exit(1);
|
|
194
|
+
}
|
|
181
195
|
}
|
|
182
196
|
}
|
|
@@ -0,0 +1,175 @@
|
|
|
1
|
+
import {Worker} from "node:worker_threads";
|
|
2
|
+
import fs from "node:fs";
|
|
3
|
+
import os from "node:os";
|
|
4
|
+
import he from "he";
|
|
5
|
+
import sanitizeHtml from "sanitize-html";
|
|
6
|
+
import {getCleanJson} from "./UtilClean.js";
|
|
7
|
+
import {ObjectWalker} from "./ObjectWalker.js";
|
|
8
|
+
import * as Uf from "./UtilFs.js";
|
|
9
|
+
import Um from "./UtilMisc.js";
|
|
10
|
+
import {Deferred, WorkerList} from "./WorkerList.js";
|
|
11
|
+
|
|
12
|
+
export class BrewCleanerHtml {
|
|
13
|
+
static _LOG_TAG = `HTML`;
|
|
14
|
+
|
|
15
|
+
static _OPTS_SANITIZE = {
|
|
16
|
+
allowedTags: [
|
|
17
|
+
// region Custom things which look like tags
|
|
18
|
+
"<$name$>",
|
|
19
|
+
// endregion
|
|
20
|
+
],
|
|
21
|
+
allowedAttributes: {},
|
|
22
|
+
};
|
|
23
|
+
|
|
24
|
+
static _getCleanFileMeta ({file}) {
|
|
25
|
+
const fileData = Uf.readJsonSync(file);
|
|
26
|
+
|
|
27
|
+
const messages = [];
|
|
28
|
+
|
|
29
|
+
const {_meta, _test} = fileData;
|
|
30
|
+
delete fileData._meta;
|
|
31
|
+
delete fileData._test;
|
|
32
|
+
|
|
33
|
+
const keyStack = [];
|
|
34
|
+
const objectStack = [];
|
|
35
|
+
|
|
36
|
+
const isInFoundryDescriptionEffect = () => {
|
|
37
|
+
if (objectStack.at(-1)?.key !== "system.description.value") return false;
|
|
38
|
+
return keyStack.at(-1) === "changes" && ["effects", "foundryEffects"].includes(keyStack.at(-2));
|
|
39
|
+
};
|
|
40
|
+
|
|
41
|
+
const fileOut = ObjectWalker.walk({
|
|
42
|
+
obj: fileData,
|
|
43
|
+
filePath: file,
|
|
44
|
+
primitiveHandlers: {
|
|
45
|
+
string: (str, {lastKey}) => {
|
|
46
|
+
if (lastKey === "value" && isInFoundryDescriptionEffect()) return str;
|
|
47
|
+
|
|
48
|
+
const clean = he.unescape(
|
|
49
|
+
sanitizeHtml(
|
|
50
|
+
str,
|
|
51
|
+
this._OPTS_SANITIZE,
|
|
52
|
+
),
|
|
53
|
+
);
|
|
54
|
+
|
|
55
|
+
if (clean !== str) {
|
|
56
|
+
const msg = `Sanitized ${keyStack.map(k => `"${k}"`).join(" -> ")}:\n${str}\n${clean}`;
|
|
57
|
+
messages.push(msg);
|
|
58
|
+
Um.info(this._LOG_TAG, msg);
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
return clean;
|
|
62
|
+
},
|
|
63
|
+
preObject: (obj) => objectStack.push(obj),
|
|
64
|
+
postObject: () => objectStack.pop(),
|
|
65
|
+
preArray: (_, {lastKey}) => keyStack.push(lastKey),
|
|
66
|
+
postArray: () => keyStack.pop(),
|
|
67
|
+
},
|
|
68
|
+
isModify: true,
|
|
69
|
+
});
|
|
70
|
+
|
|
71
|
+
const out = {$schema: fileOut.$schema, _meta, _test};
|
|
72
|
+
Object.assign(out, fileOut);
|
|
73
|
+
|
|
74
|
+
return {
|
|
75
|
+
messages,
|
|
76
|
+
out,
|
|
77
|
+
};
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
static async _pUpdateDir (dir) {
|
|
81
|
+
Uf.listJsonFiles(dir)
|
|
82
|
+
.forEach(file => {
|
|
83
|
+
const {messages, out} = this._getCleanFileMeta({file});
|
|
84
|
+
if (!messages?.length) return;
|
|
85
|
+
|
|
86
|
+
fs.writeFileSync(file, getCleanJson(out));
|
|
87
|
+
});
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
static async pRun () {
|
|
91
|
+
await Uf.pRunOnDirs(
|
|
92
|
+
async (dir) => {
|
|
93
|
+
Um.info(this._LOG_TAG, `Sanitizing HTML in dir "${dir}"...`);
|
|
94
|
+
await this._pUpdateDir(dir);
|
|
95
|
+
},
|
|
96
|
+
{
|
|
97
|
+
isSerial: true,
|
|
98
|
+
},
|
|
99
|
+
);
|
|
100
|
+
Um.info(this._LOG_TAG, "Done!");
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
static getFileMessages ({file}) {
|
|
104
|
+
return this._getCleanFileMeta({file});
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
static async pGetErrorsOnDirsWorkers ({isFailFast = false} = {}) {
|
|
108
|
+
Um.info(this._LOG_TAG, `Testing for HTML...`);
|
|
109
|
+
|
|
110
|
+
const cntWorkers = Math.max(1, os.cpus().length - 1);
|
|
111
|
+
|
|
112
|
+
const messages = [];
|
|
113
|
+
|
|
114
|
+
const fileQueue = [];
|
|
115
|
+
Uf.runOnDirs((dir) => fileQueue.push(...Uf.listJsonFiles(dir)));
|
|
116
|
+
|
|
117
|
+
const workerList = new WorkerList();
|
|
118
|
+
|
|
119
|
+
let cntFailures = 0;
|
|
120
|
+
const workers = [...new Array(cntWorkers)]
|
|
121
|
+
.map(() => {
|
|
122
|
+
const worker = new Worker(new URL("./BrewCleanerHtmlWorker.js", import.meta.url));
|
|
123
|
+
|
|
124
|
+
worker.on("message", (msg) => {
|
|
125
|
+
switch (msg.type) {
|
|
126
|
+
case "ready":
|
|
127
|
+
case "done": {
|
|
128
|
+
if (msg.payload.isError) {
|
|
129
|
+
messages.push(...msg.payload.messages);
|
|
130
|
+
|
|
131
|
+
if (isFailFast) workers.forEach(worker => worker.postMessage({type: "cancel"}));
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
if (worker.dIsActive) worker.dIsActive.resolve();
|
|
135
|
+
workerList.add(worker);
|
|
136
|
+
|
|
137
|
+
break;
|
|
138
|
+
}
|
|
139
|
+
}
|
|
140
|
+
});
|
|
141
|
+
|
|
142
|
+
worker.on("error", e => {
|
|
143
|
+
console.error(e);
|
|
144
|
+
cntFailures++;
|
|
145
|
+
});
|
|
146
|
+
|
|
147
|
+
worker.postMessage({
|
|
148
|
+
type: "init",
|
|
149
|
+
payload: {},
|
|
150
|
+
});
|
|
151
|
+
|
|
152
|
+
return worker;
|
|
153
|
+
});
|
|
154
|
+
|
|
155
|
+
while (fileQueue.length) {
|
|
156
|
+
if (isFailFast && messages.length) break;
|
|
157
|
+
|
|
158
|
+
const file = fileQueue.shift();
|
|
159
|
+
const worker = await workerList.get();
|
|
160
|
+
|
|
161
|
+
worker.dIsActive = new Deferred();
|
|
162
|
+
worker.postMessage({
|
|
163
|
+
type: "work",
|
|
164
|
+
payload: {
|
|
165
|
+
file,
|
|
166
|
+
},
|
|
167
|
+
});
|
|
168
|
+
}
|
|
169
|
+
|
|
170
|
+
await Promise.all(workers.map(it => it.dIsActive?.promise));
|
|
171
|
+
await Promise.all(workers.map(it => it.terminate()));
|
|
172
|
+
|
|
173
|
+
return {messages, isUnknownError: !!cntFailures};
|
|
174
|
+
}
|
|
175
|
+
}
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
import {isMainThread, parentPort} from "node:worker_threads";
|
|
2
|
+
import {BrewCleanerHtml} from "./BrewCleanerHtml.js";
|
|
3
|
+
|
|
4
|
+
if (isMainThread) throw new Error(`Worker must not be started in main thread!`);
|
|
5
|
+
|
|
6
|
+
let isCancelled = false;
|
|
7
|
+
|
|
8
|
+
parentPort
|
|
9
|
+
.on("message", async msg => {
|
|
10
|
+
switch (msg.type) {
|
|
11
|
+
case "init": {
|
|
12
|
+
parentPort.postMessage({
|
|
13
|
+
type: "ready",
|
|
14
|
+
payload: {},
|
|
15
|
+
});
|
|
16
|
+
|
|
17
|
+
break;
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
case "cancel": {
|
|
21
|
+
isCancelled = true;
|
|
22
|
+
break;
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
case "work": {
|
|
26
|
+
if (isCancelled) {
|
|
27
|
+
parentPort.postMessage({
|
|
28
|
+
type: "done",
|
|
29
|
+
payload: {},
|
|
30
|
+
});
|
|
31
|
+
return;
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
const {messages = []} = BrewCleanerHtml.getFileMessages({file: msg.payload.file});
|
|
35
|
+
|
|
36
|
+
parentPort.postMessage({
|
|
37
|
+
type: "done",
|
|
38
|
+
payload: {
|
|
39
|
+
isError: !!messages.length,
|
|
40
|
+
messages,
|
|
41
|
+
},
|
|
42
|
+
});
|
|
43
|
+
|
|
44
|
+
break;
|
|
45
|
+
}
|
|
46
|
+
}
|
|
47
|
+
});
|
|
@@ -0,0 +1,207 @@
|
|
|
1
|
+
import fs from "node:fs";
|
|
2
|
+
import {BrewTesterBase} from "./BrewTesterBase.js";
|
|
3
|
+
import {DataTester, DataTesterBase, BraceCheck, EscapeCharacterCheck} from "../TestData.js";
|
|
4
|
+
import * as Uf from "../UtilFs.js";
|
|
5
|
+
import {ObjectWalker} from "../ObjectWalker.js";
|
|
6
|
+
import {UtilSource} from "../UtilSource.js";
|
|
7
|
+
import Um from "../UtilMisc.js";
|
|
8
|
+
|
|
9
|
+
class _CopySourceCheck extends DataTesterBase {
|
|
10
|
+
static _FileState = class {
|
|
11
|
+
sources;
|
|
12
|
+
dependencies;
|
|
13
|
+
internalCopies;
|
|
14
|
+
|
|
15
|
+
constructor ({contents}) {
|
|
16
|
+
this.sources = new Set(
|
|
17
|
+
(contents._meta?.sources?.map(src => src?.json) || [])
|
|
18
|
+
.filter(Boolean),
|
|
19
|
+
);
|
|
20
|
+
this.dependencies = Object.fromEntries(
|
|
21
|
+
Object.entries(contents._meta?.dependencies || {})
|
|
22
|
+
.map(([prop, arr]) => [prop, new Set(arr)]),
|
|
23
|
+
);
|
|
24
|
+
this.internalCopies = new Set(contents._meta?.internalCopies || []);
|
|
25
|
+
}
|
|
26
|
+
};
|
|
27
|
+
|
|
28
|
+
registerParsedFileCheckers (parsedJsonChecker) {
|
|
29
|
+
parsedJsonChecker.registerFileHandler(this);
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
handleFile (file, contents) {
|
|
33
|
+
if (!file.includes("Kobold Press; Scarlet Citadel.json")) return;
|
|
34
|
+
|
|
35
|
+
const fileState = new this.constructor._FileState({contents});
|
|
36
|
+
|
|
37
|
+
Object.entries(contents)
|
|
38
|
+
.forEach(([prop, arr]) => {
|
|
39
|
+
if (prop.startsWith("_")) return;
|
|
40
|
+
if (!(arr instanceof Array)) return;
|
|
41
|
+
|
|
42
|
+
arr.forEach(ent => {
|
|
43
|
+
const propStack = [prop];
|
|
44
|
+
const inlineDependencies = new Set();
|
|
45
|
+
|
|
46
|
+
ObjectWalker.walk({
|
|
47
|
+
obj: ent,
|
|
48
|
+
filePath: file,
|
|
49
|
+
primitiveHandlers: {
|
|
50
|
+
preObject: this._onPreObject.bind(this, {propStack, inlineDependencies}),
|
|
51
|
+
object: this._checkObject.bind(this, {fileState, propStack, inlineDependencies}),
|
|
52
|
+
postObject: this._onPostObject.bind(this, {propStack, inlineDependencies}),
|
|
53
|
+
},
|
|
54
|
+
});
|
|
55
|
+
});
|
|
56
|
+
});
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
_onPreObject ({propStack, inlineDependencies}, obj) {
|
|
60
|
+
if (obj.type !== "statblockInline") return;
|
|
61
|
+
|
|
62
|
+
propStack.push(obj.dataType);
|
|
63
|
+
(obj.dependencies || []).forEach(dep => inlineDependencies.add(dep));
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
_onPostObject ({propStack, inlineDependencies}, obj) {
|
|
67
|
+
if (obj.type !== "statblockInline") return;
|
|
68
|
+
|
|
69
|
+
propStack.pop();
|
|
70
|
+
inlineDependencies.clear();
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
_checkObject ({fileState, propStack, inlineDependencies}, obj, {filePath}) {
|
|
74
|
+
if (!obj._copy?.source) return;
|
|
75
|
+
|
|
76
|
+
const prop = propStack.at(-1);
|
|
77
|
+
const sourceCopy = obj._copy.source;
|
|
78
|
+
|
|
79
|
+
// Classes/subclasses have an alternate structure.
|
|
80
|
+
if (["class", "subclass"].includes(prop) && UtilSource.isSiteSource(sourceCopy)) {
|
|
81
|
+
const classNameLower = obj._copy.className?.toLowerCase();
|
|
82
|
+
if (
|
|
83
|
+
fileState.dependencies[prop]?.has(classNameLower)
|
|
84
|
+
|| inlineDependencies.has(classNameLower)
|
|
85
|
+
) return;
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
// If a root entity, i.e. not in a `statblockInline`, allow internal copies.
|
|
89
|
+
if (
|
|
90
|
+
propStack.length === 1
|
|
91
|
+
&& fileState.internalCopies.has(prop)
|
|
92
|
+
&& fileState.sources.has(sourceCopy)
|
|
93
|
+
) return;
|
|
94
|
+
|
|
95
|
+
if (
|
|
96
|
+
fileState.dependencies[prop]?.has(sourceCopy)
|
|
97
|
+
|| inlineDependencies.has(sourceCopy)
|
|
98
|
+
) return;
|
|
99
|
+
|
|
100
|
+
this._addMessage(`Entity "${propStack.join(" -> ")}" "${obj.name}" "_copy" source "${sourceCopy}" did not match sources found in dependencies in file "${filePath}"\n`);
|
|
101
|
+
}
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
class _ImageUrlCheck extends DataTesterBase {
|
|
105
|
+
static _RE_IMG_PATH = /^(?<type>img|pdf)\/(?<source>[^/]+)\//;
|
|
106
|
+
|
|
107
|
+
static _FileState = class {
|
|
108
|
+
sources;
|
|
109
|
+
|
|
110
|
+
constructor ({contents}) {
|
|
111
|
+
this.sources = new Set(
|
|
112
|
+
[
|
|
113
|
+
...(contents._meta?.sources?.map(src => src?.json) || [])
|
|
114
|
+
.filter(Boolean)
|
|
115
|
+
.map(srcJson => srcJson.replace(/:/g, "")),
|
|
116
|
+
...(contents._test?.additionalImageSources || [])
|
|
117
|
+
.map(srcJson => srcJson.replace(/:/g, "")),
|
|
118
|
+
],
|
|
119
|
+
);
|
|
120
|
+
}
|
|
121
|
+
};
|
|
122
|
+
|
|
123
|
+
constructor ({imgRepoName, urlPrefixExpected}) {
|
|
124
|
+
super();
|
|
125
|
+
this._imgRepoName = imgRepoName;
|
|
126
|
+
this._urlPrefixExpected = urlPrefixExpected;
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
registerParsedFileCheckers (parsedJsonChecker) {
|
|
130
|
+
parsedJsonChecker.registerFileHandler(this);
|
|
131
|
+
}
|
|
132
|
+
|
|
133
|
+
handleFile (file, contents) {
|
|
134
|
+
const fileState = new this.constructor._FileState({contents});
|
|
135
|
+
|
|
136
|
+
ObjectWalker.walk({
|
|
137
|
+
obj: contents,
|
|
138
|
+
filePath: file,
|
|
139
|
+
primitiveHandlers: {
|
|
140
|
+
object: this._checkObject.bind(this, {fileState}),
|
|
141
|
+
},
|
|
142
|
+
});
|
|
143
|
+
}
|
|
144
|
+
|
|
145
|
+
_checkObject ({fileState}, obj, {filePath}) {
|
|
146
|
+
if (obj.type !== "image" || obj.href?.type !== "external" || !obj.href?.url) return;
|
|
147
|
+
|
|
148
|
+
const {url} = obj.href;
|
|
149
|
+
if (!url.toLowerCase().startsWith(this._urlPrefixExpected.toLowerCase())) return;
|
|
150
|
+
|
|
151
|
+
const mPath = this.constructor._RE_IMG_PATH.exec(url.slice(this._urlPrefixExpected.length));
|
|
152
|
+
if (!mPath) {
|
|
153
|
+
this._addMessage(`Unknown "${this._imgRepoName}" URL pattern in file "${filePath}": "${url}"\n`);
|
|
154
|
+
return;
|
|
155
|
+
}
|
|
156
|
+
|
|
157
|
+
const {source, type} = mPath.groups;
|
|
158
|
+
if (fileState.sources.has(source)) return;
|
|
159
|
+
|
|
160
|
+
this._addMessage(`Image source part "${source}" in "${this._imgRepoName}" ${type} URL did not match sources found in file "_meta" or "_test" in file "${filePath}": "${url}"\n`);
|
|
161
|
+
}
|
|
162
|
+
}
|
|
163
|
+
|
|
164
|
+
export class BrewTesterFileContents extends BrewTesterBase {
|
|
165
|
+
_LOG_TAG = "FILE_CONTENTS";
|
|
166
|
+
|
|
167
|
+
constructor ({imgRepoName, urlPrefixExpected, pathErrorLog} = {}) {
|
|
168
|
+
super();
|
|
169
|
+
this._imgRepoName = imgRepoName;
|
|
170
|
+
this._urlPrefixExpected = urlPrefixExpected;
|
|
171
|
+
this._pathErrorLog = pathErrorLog || "test-data.error.log";
|
|
172
|
+
}
|
|
173
|
+
|
|
174
|
+
async _pRun () {
|
|
175
|
+
if (!this._imgRepoName) throw new Error(`Image repo name was required!`);
|
|
176
|
+
if (!this._urlPrefixExpected) throw new Error(`Expected URL prefix was required!`);
|
|
177
|
+
|
|
178
|
+
Um.info(this._LOG_TAG, `Running checks for image repo "${this._imgRepoName}" with URL prefix "${this._urlPrefixExpected}"...`);
|
|
179
|
+
|
|
180
|
+
const dataTesters = [
|
|
181
|
+
new BraceCheck(),
|
|
182
|
+
new EscapeCharacterCheck(),
|
|
183
|
+
new _ImageUrlCheck({imgRepoName: this._imgRepoName, urlPrefixExpected: this._urlPrefixExpected}),
|
|
184
|
+
new _CopySourceCheck(),
|
|
185
|
+
];
|
|
186
|
+
DataTester.register({dataTesters});
|
|
187
|
+
|
|
188
|
+
await Uf.pRunOnDirs(
|
|
189
|
+
async (dir) => {
|
|
190
|
+
Um.info(this._LOG_TAG, `Checking dir "${dir}"...`);
|
|
191
|
+
await DataTester.pRun(dir, dataTesters);
|
|
192
|
+
},
|
|
193
|
+
{
|
|
194
|
+
isSerial: true,
|
|
195
|
+
},
|
|
196
|
+
);
|
|
197
|
+
|
|
198
|
+
const outMessage = DataTester.getLogReport(dataTesters);
|
|
199
|
+
if (!outMessage) {
|
|
200
|
+
Um.info(this._LOG_TAG, `Complete.`);
|
|
201
|
+
return;
|
|
202
|
+
}
|
|
203
|
+
|
|
204
|
+
fs.writeFileSync(this._pathErrorLog, outMessage, "utf-8");
|
|
205
|
+
throw new Error(`Checks failed! See "${this._pathErrorLog}" and logs above.`);
|
|
206
|
+
}
|
|
207
|
+
}
|
|
@@ -8,6 +8,11 @@ export class BrewTesterFileNames extends BrewTesterBase {
|
|
|
8
8
|
|
|
9
9
|
static _RE_NAME_FORMAT = /^[^;]+; .+\.json$/;
|
|
10
10
|
|
|
11
|
+
constructor ({reNameFormat = null} = {}) {
|
|
12
|
+
super();
|
|
13
|
+
this._reNameFormat = reNameFormat || this.constructor._RE_NAME_FORMAT;
|
|
14
|
+
}
|
|
15
|
+
|
|
11
16
|
async _pRun () {
|
|
12
17
|
Um.info(this._LOG_TAG, `Testing for incorrect file names...`);
|
|
13
18
|
|
|
@@ -24,8 +29,8 @@ export class BrewTesterFileNames extends BrewTesterBase {
|
|
|
24
29
|
|
|
25
30
|
errors.push(
|
|
26
31
|
...filenames
|
|
27
|
-
.filter(it => !this.
|
|
28
|
-
.map(it => `Filename did not match expected pattern "${this.
|
|
32
|
+
.filter(it => !this._reNameFormat.test(it))
|
|
33
|
+
.map(it => `Filename did not match expected pattern "${this._reNameFormat.toString()}" extension: ${it}`),
|
|
29
34
|
);
|
|
30
35
|
});
|
|
31
36
|
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
import fs from "node:fs";
|
|
2
|
+
import {BrewTesterBase} from "./BrewTesterBase.js";
|
|
3
|
+
import Um from "../UtilMisc.js";
|
|
4
|
+
|
|
5
|
+
export class BrewTesterImgDirectories extends BrewTesterBase {
|
|
6
|
+
_LOG_TAG = "IMG_DIR";
|
|
7
|
+
|
|
8
|
+
constructor ({dirAllowlist = null, pathImgDir = "_img"} = {}) {
|
|
9
|
+
super();
|
|
10
|
+
this._dirAllowlist = new Set(dirAllowlist || []);
|
|
11
|
+
this._pathImgDir = pathImgDir;
|
|
12
|
+
}
|
|
13
|
+
|
|
14
|
+
async _pRun () {
|
|
15
|
+
if (!this._dirAllowlist.size) throw new Error(`Directory allowlist was required!`);
|
|
16
|
+
|
|
17
|
+
if (!fs.existsSync(this._pathImgDir)) {
|
|
18
|
+
Um.info(this._LOG_TAG, `Directory "${this._pathImgDir}" was not found; skipping.`);
|
|
19
|
+
return;
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
const extraDirs = fs.readdirSync(this._pathImgDir)
|
|
23
|
+
.filter(dir => !this._dirAllowlist.has(dir));
|
|
24
|
+
|
|
25
|
+
if (!extraDirs.length) {
|
|
26
|
+
Um.info(this._LOG_TAG, `No unexpected entries found in "${this._pathImgDir}".`);
|
|
27
|
+
return;
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
throw new Error(`Extra directories found in "${this._pathImgDir}":\n${extraDirs.map(d => `\t${d}`).join("\n")}`);
|
|
31
|
+
}
|
|
32
|
+
}
|
package/lib/BrewTester.js
CHANGED
|
@@ -3,11 +3,15 @@ import {BrewTesterFileLocations} from "./BrewTester/BrewTesterFileLocations.js";
|
|
|
3
3
|
import {BrewTesterFileNames} from "./BrewTester/BrewTesterFileNames.js";
|
|
4
4
|
import {BrewTesterFileProps} from "./BrewTester/BrewTesterFileProps.js";
|
|
5
5
|
import {BrewTesterEdition} from "./BrewTester/BrewTesterEdition.js";
|
|
6
|
+
import {BrewTesterFileContents} from "./BrewTester/BrewTesterFileContents.js";
|
|
7
|
+
import {BrewTesterImgDirectories} from "./BrewTester/BrewTesterImgDirectories.js";
|
|
6
8
|
|
|
7
9
|
export class BrewTester {
|
|
8
10
|
static async pTestJson ({mode, filepath, dir}) { return (new BrewTesterJson({mode, filepath, dir})).pRun(); }
|
|
9
11
|
static pTestFileLocations () { return (new BrewTesterFileLocations()).pRun(); }
|
|
10
|
-
static pTestFileNames () { return (new BrewTesterFileNames()).pRun(); }
|
|
12
|
+
static pTestFileNames ({reNameFormat} = {}) { return (new BrewTesterFileNames({reNameFormat})).pRun(); }
|
|
11
13
|
static pTestFileProps () { return (new BrewTesterFileProps()).pRun(); }
|
|
14
|
+
static pTestFileContents ({imgRepoName, urlPrefixExpected, pathErrorLog}) { return (new BrewTesterFileContents({imgRepoName, urlPrefixExpected, pathErrorLog})).pRun(); }
|
|
15
|
+
static pTestImgDirectories ({dirAllowlist, pathImgDir} = {}) { return (new BrewTesterImgDirectories({dirAllowlist, pathImgDir})).pRun(); }
|
|
12
16
|
static pTestEditionSources () { return (new BrewTesterEdition()).pRun(); }
|
|
13
17
|
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "5etools-utils",
|
|
3
|
-
"version": "0.15.
|
|
3
|
+
"version": "0.15.4",
|
|
4
4
|
"description": "Shared utilities for the 5etools ecosystem.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "lib/Api.js",
|
|
@@ -12,6 +12,8 @@
|
|
|
12
12
|
"lib": "./lib"
|
|
13
13
|
},
|
|
14
14
|
"bin": {
|
|
15
|
+
"clean-html": "bin/clean-html.js",
|
|
16
|
+
"test-file-contents": "bin/test-file-contents.js",
|
|
15
17
|
"test-json-brew": "bin/test-json-brew.js",
|
|
16
18
|
"test-json-ua": "bin/test-json-ua.js",
|
|
17
19
|
"test-file-names": "bin/test-file-names.js",
|
|
@@ -43,8 +45,10 @@
|
|
|
43
45
|
"ajv-formats": "^3.0.1",
|
|
44
46
|
"commander": "^14.0.3",
|
|
45
47
|
"hasha": "^7.0.0",
|
|
48
|
+
"he": "^1.2.0",
|
|
46
49
|
"json-source-map": "^0.6.1",
|
|
47
|
-
"number-precision": "^1.6.0"
|
|
50
|
+
"number-precision": "^1.6.0",
|
|
51
|
+
"sanitize-html": "^2.17.1"
|
|
48
52
|
},
|
|
49
53
|
"devDependencies": {
|
|
50
54
|
"@eslint/js": "^10.0.1",
|