5etools-utils 0.3.0 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/lib/Api.js +11 -1
- package/lib/BrewCleaner.js +126 -0
- package/lib/BrewIndexGenerator.js +173 -0
- package/lib/BrewTimestamper.js +102 -0
- package/lib/TestJson.js +43 -19
- package/lib/UtilClean.js +35 -0
- package/lib/UtilMisc.js +36 -17
- package/lib/UtilSources.js +200 -0
- package/package.json +5 -2
- package/schema/brew/bestiary/fluff-index.json +11 -0
- package/schema/brew/bestiary/index.json +11 -0
- package/schema/brew/changelog.json +35 -0
- package/schema/brew/class/index.json +11 -0
- package/schema/brew/homebrew.json +2 -2
- package/schema/brew/life.json +17 -0
- package/schema/brew/spells/fluff-index.json +11 -0
- package/schema/brew/spells/index.json +11 -0
- package/schema/brew-fast/bestiary/fluff-index.json +11 -0
- package/schema/brew-fast/bestiary/index.json +11 -0
- package/schema/brew-fast/changelog.json +35 -0
- package/schema/brew-fast/class/index.json +11 -0
- package/schema/brew-fast/homebrew.json +2 -2
- package/schema/brew-fast/life.json +17 -0
- package/schema/brew-fast/spells/fluff-index.json +11 -0
- package/schema/brew-fast/spells/index.json +11 -0
- package/schema/site/bestiary/fluff-index.json +11 -0
- package/schema/site/bestiary/index.json +11 -0
- package/schema/site/changelog.json +35 -0
- package/schema/site/class/index.json +11 -0
- package/schema/site/homebrew.json +2 -2
- package/schema/site/life.json +17 -0
- package/schema/site/spells/fluff-index.json +11 -0
- package/schema/site/spells/index.json +11 -0
- package/schema/site-fast/bestiary/fluff-index.json +11 -0
- package/schema/site-fast/bestiary/index.json +11 -0
- package/schema/site-fast/changelog.json +35 -0
- package/schema/site-fast/class/index.json +11 -0
- package/schema/site-fast/homebrew.json +2 -2
- package/schema/site-fast/life.json +17 -0
- package/schema/site-fast/spells/fluff-index.json +11 -0
- package/schema/site-fast/spells/index.json +11 -0
package/lib/Api.js
CHANGED
|
@@ -1,9 +1,19 @@
|
|
|
1
1
|
import {JsonTester} from "./TestJson.js";
|
|
2
|
-
import
|
|
2
|
+
import Um from "./UtilMisc.js";
|
|
3
3
|
import * as Uf from "./UtilFs.js";
|
|
4
|
+
import {BrewIndexGenerator} from "./BrewIndexGenerator.js";
|
|
5
|
+
import {BrewCleaner} from "./BrewCleaner.js";
|
|
6
|
+
import {BrewTimestamper} from "./BrewTimestamper.js";
|
|
7
|
+
import {SITE_SOURCES} from "./UtilSources.js";
|
|
8
|
+
import {getCleanJson} from "./UtilClean.js";
|
|
4
9
|
|
|
5
10
|
export {
|
|
6
11
|
JsonTester,
|
|
7
12
|
Um,
|
|
8
13
|
Uf,
|
|
14
|
+
BrewIndexGenerator,
|
|
15
|
+
BrewCleaner,
|
|
16
|
+
BrewTimestamper,
|
|
17
|
+
SITE_SOURCES,
|
|
18
|
+
getCleanJson,
|
|
9
19
|
};
|
|
@@ -0,0 +1,126 @@
|
|
|
1
|
+
// Adapted from 5etools `clean-jsons.js`
|
|
2
|
+
// ===
|
|
3
|
+
|
|
4
|
+
import * as fs from "fs";
|
|
5
|
+
import * as Uf from "./UtilFs.js";
|
|
6
|
+
import Um from "./UtilMisc.js";
|
|
7
|
+
import {getCleanJson} from "./UtilClean.js";
|
|
8
|
+
import {SITE_SOURCES} from "./UtilSources.js";
|
|
9
|
+
|
|
10
|
+
class BrewCleaner {
|
|
11
|
+
static _IS_FAIL_SLOW = !!process.env.FAIL_SLOW;
|
|
12
|
+
static _SITE_SOURCES = new Set(SITE_SOURCES);
|
|
13
|
+
|
|
14
|
+
static _RUN_TIMESTAMP = Math.floor(Date.now() / 1000);
|
|
15
|
+
static _MAX_TIMESTAMP = 9999999999;
|
|
16
|
+
|
|
17
|
+
static _CONTENT_KEY_BLOCKLIST = new Set(["$schema", "_meta", "siteVersion"]);
|
|
18
|
+
|
|
19
|
+
static _RE_INVALID_WINDOWS_CHARS = /[<>:"/\\|?*]/g;
|
|
20
|
+
|
|
21
|
+
static _ALL_SOURCES_JSON_LOWER = new Set();
|
|
22
|
+
|
|
23
|
+
static _cleanFolder (folder) {
|
|
24
|
+
const ALL_ERRORS = [];
|
|
25
|
+
|
|
26
|
+
const files = Uf.listJsonFiles(folder);
|
|
27
|
+
for (const file of files) {
|
|
28
|
+
let contents = Uf.readJSON(file);
|
|
29
|
+
|
|
30
|
+
if (this._RE_INVALID_WINDOWS_CHARS.test(file.split("/").slice(1).join("/"))) {
|
|
31
|
+
ALL_ERRORS.push(`${file} contained invalid characters!`);
|
|
32
|
+
if (!this._IS_FAIL_SLOW) break;
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
if (!file.endsWith(".json")) {
|
|
36
|
+
ALL_ERRORS.push(`${file} had invalid extension! Should be ".json" (case-sensitive).`);
|
|
37
|
+
if (!this._IS_FAIL_SLOW) break;
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
// region clean
|
|
41
|
+
// Ensure _meta is at the top of the file
|
|
42
|
+
const tmp = {$schema: contents.$schema, _meta: contents._meta};
|
|
43
|
+
delete contents.$schema;
|
|
44
|
+
delete contents._meta;
|
|
45
|
+
Object.assign(tmp, contents);
|
|
46
|
+
contents = tmp;
|
|
47
|
+
|
|
48
|
+
if (contents._meta.dateAdded == null) {
|
|
49
|
+
Um.warn(`TIMESTAMPS`, `\tFile "${file}" did not have "dateAdded"! Adding one...`);
|
|
50
|
+
contents._meta.dateAdded = this._RUN_TIMESTAMP;
|
|
51
|
+
} else if (contents._meta.dateAdded > this._MAX_TIMESTAMP) {
|
|
52
|
+
Um.warn(`TIMESTAMPS`, `\tFile "${file}" had a "dateAdded" in milliseconds! Converting to seconds...`);
|
|
53
|
+
contents._meta.dateAdded = Math.round(contents._meta.dateAdded / 1000);
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
if (contents._meta.dateLastModified == null) {
|
|
57
|
+
Um.warn(`TIMESTAMPS`, `\tFile "${file}" did not have "dateLastModified"! Adding one...`);
|
|
58
|
+
contents._meta.dateLastModified = this._RUN_TIMESTAMP;
|
|
59
|
+
} else if (contents._meta.dateLastModified > this._MAX_TIMESTAMP) {
|
|
60
|
+
Um.warn(`TIMESTAMPS`, `\tFile "${file}" had a "dateLastModified" in milliseconds! Converting to seconds...`);
|
|
61
|
+
contents._meta.dateLastModified = Math.round(contents._meta.dateLastModified / 1000);
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
(contents._meta.sources || []).forEach(source => {
|
|
65
|
+
if (source.version != null) return;
|
|
66
|
+
Um.warn(`VERSION`, `\tFile "${file}" source "${source.json}" did not have "version"! Adding one...`);
|
|
67
|
+
source.version = "unknown";
|
|
68
|
+
});
|
|
69
|
+
// endregion
|
|
70
|
+
|
|
71
|
+
// region test
|
|
72
|
+
const docSourcesJson = contents._meta.sources.map(src => src.json);
|
|
73
|
+
const duplicateSourcesJson = docSourcesJson.filter(src => this._ALL_SOURCES_JSON_LOWER.has(src.toLowerCase()));
|
|
74
|
+
if (duplicateSourcesJson.length) {
|
|
75
|
+
ALL_ERRORS.push(`${file} :: "json" source${duplicateSourcesJson.length === 1 ? "" : "s"} exist in other documents; sources were: ${duplicateSourcesJson.map(src => `"${src}"`).join(", ")}`);
|
|
76
|
+
}
|
|
77
|
+
docSourcesJson.forEach(src => this._ALL_SOURCES_JSON_LOWER.add(src.toLowerCase()));
|
|
78
|
+
|
|
79
|
+
const validSources = new Set(docSourcesJson);
|
|
80
|
+
validSources.add("UAClassFeatureVariants"); // Allow CFV UA sources
|
|
81
|
+
|
|
82
|
+
Object.keys(contents)
|
|
83
|
+
.filter(k => !this._CONTENT_KEY_BLOCKLIST.has(k))
|
|
84
|
+
.forEach(k => {
|
|
85
|
+
const data = contents[k];
|
|
86
|
+
|
|
87
|
+
if (!(data instanceof Array) || !data.forEach) throw new Error(`File "${k}" data was not an array!`);
|
|
88
|
+
|
|
89
|
+
if (!data.length) throw new Error(`File "${k}" array is empty!`);
|
|
90
|
+
|
|
91
|
+
data.forEach(it => {
|
|
92
|
+
const source = it.source || (it.inherits ? it.inherits.source : null);
|
|
93
|
+
if (!source) return ALL_ERRORS.push(`${file} :: ${k} :: "${it.name || it.id}" had no source!`);
|
|
94
|
+
if (!validSources.has(source) && !this._SITE_SOURCES.has(source)) return ALL_ERRORS.push(`${file} :: ${k} :: "${it.name || it.id}" source "${source}" was not in _meta`);
|
|
95
|
+
});
|
|
96
|
+
});
|
|
97
|
+
// endregion
|
|
98
|
+
|
|
99
|
+
if (!this._IS_FAIL_SLOW && ALL_ERRORS.length) break;
|
|
100
|
+
|
|
101
|
+
Um.info(`CLEANER`, `\t- "${file}"...`);
|
|
102
|
+
contents = getCleanJson(contents);
|
|
103
|
+
|
|
104
|
+
fs.writeFileSync(file, contents);
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
if (ALL_ERRORS.length) {
|
|
108
|
+
ALL_ERRORS.forEach(e => console.error(e));
|
|
109
|
+
throw new Error(`Errors were found. See above.`);
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
return files.length;
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
static run () {
|
|
116
|
+
let totalFiles = 0;
|
|
117
|
+
Uf.runOnDirs((dir) => {
|
|
118
|
+
Um.info(`CLEANER`, `Cleaning dir "${dir}"...`);
|
|
119
|
+
totalFiles += this._cleanFolder(dir);
|
|
120
|
+
});
|
|
121
|
+
|
|
122
|
+
Um.info(`CLEANER`, `Cleaning complete. Cleaned ${totalFiles} file${totalFiles === 1 ? "" : "s"}.`);
|
|
123
|
+
}
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
export {BrewCleaner};
|
|
@@ -0,0 +1,173 @@
|
|
|
1
|
+
import * as fs from "fs";
|
|
2
|
+
import * as Uf from "./UtilFs.js";
|
|
3
|
+
import Um from "./UtilMisc.js";
|
|
4
|
+
|
|
5
|
+
class _BrewIndex {
|
|
6
|
+
static _FILE_PATH;
|
|
7
|
+
static _DISPLAY_NAME;
|
|
8
|
+
|
|
9
|
+
_index = {};
|
|
10
|
+
|
|
11
|
+
doWrite () {
|
|
12
|
+
Um.info(`INDEX`, `Saving ${this.constructor._DISPLAY_NAME} index to ${this.constructor._FILE_PATH}`);
|
|
13
|
+
fs.writeFileSync(`./${this.constructor._FILE_PATH}`, JSON.stringify(this._index), "utf-8");
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
/** @abstract */
|
|
17
|
+
addToIndex (fileInfo) { throw new Error("Unimplemented!"); }
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
class _BrewIndexTimestamps extends _BrewIndex {
|
|
21
|
+
static _FILE_PATH = "_generated/index-timestamps.json";
|
|
22
|
+
static _DISPLAY_NAME = "timestamp";
|
|
23
|
+
|
|
24
|
+
addToIndex (fileInfo) {
|
|
25
|
+
this._index[fileInfo.cleanName] = {
|
|
26
|
+
a: fileInfo.contents._meta.dateAdded,
|
|
27
|
+
m: fileInfo.contents._meta.dateLastModified,
|
|
28
|
+
};
|
|
29
|
+
}
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
class _BrewIndexProps extends _BrewIndex {
|
|
33
|
+
static _FILE_PATH = "_generated/index-props.json";
|
|
34
|
+
static _DISPLAY_NAME = "prop";
|
|
35
|
+
|
|
36
|
+
addToIndex (fileInfo) {
|
|
37
|
+
Object.keys(fileInfo.contents)
|
|
38
|
+
.filter(it => !it.startsWith("_") && it !== "$schema")
|
|
39
|
+
.forEach(k => {
|
|
40
|
+
(this._index[k] = this._index[k] || {})[fileInfo.cleanName] = fileInfo.folder;
|
|
41
|
+
});
|
|
42
|
+
|
|
43
|
+
Object.keys(fileInfo.contents._meta.includes || {})
|
|
44
|
+
.forEach(k => {
|
|
45
|
+
(this._index[k] = this._index[k] || {})[fileInfo.cleanName] = fileInfo.folder;
|
|
46
|
+
});
|
|
47
|
+
}
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
class _BrewIndexSources extends _BrewIndex {
|
|
51
|
+
static _FILE_PATH = "_generated/index-sources.json";
|
|
52
|
+
static _DISPLAY_NAME = "source";
|
|
53
|
+
|
|
54
|
+
addToIndex (fileInfo) {
|
|
55
|
+
(fileInfo.contents._meta.sources || [])
|
|
56
|
+
.forEach(src => {
|
|
57
|
+
if (this._index[src.json]) throw new Error(`${fileInfo.name} source "${src.json}" was already in ${this._index[src.json]}`);
|
|
58
|
+
this._index[src.json] = fileInfo.cleanName;
|
|
59
|
+
});
|
|
60
|
+
}
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
class _BrewIndexMeta extends _BrewIndex {
|
|
64
|
+
static _FILE_PATH = "_generated/index-meta.json";
|
|
65
|
+
static _DISPLAY_NAME = "meta";
|
|
66
|
+
|
|
67
|
+
addToIndex (fileInfo) {
|
|
68
|
+
if (!fileInfo.contents._meta.sources?.length) return;
|
|
69
|
+
|
|
70
|
+
const fileName = fileInfo.name.split("/").slice(1).join("/");
|
|
71
|
+
|
|
72
|
+
if (this._index[fileName]) throw new Error(`Filename "${fileName}" was already in the index!`);
|
|
73
|
+
this._index[fileName] = {
|
|
74
|
+
// name
|
|
75
|
+
n: fileInfo.contents._meta.sources.map(it => it.full).filter(Boolean),
|
|
76
|
+
// abbreviation
|
|
77
|
+
a: fileInfo.contents._meta.sources.map(it => it.abbreviation).filter(Boolean),
|
|
78
|
+
// status
|
|
79
|
+
s: fileInfo.contents._meta.status,
|
|
80
|
+
};
|
|
81
|
+
}
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
class BrewIndexGenerator {
|
|
85
|
+
static _DIR_TO_PRIMARY_PROP = {
|
|
86
|
+
"creature": [
|
|
87
|
+
"monster",
|
|
88
|
+
],
|
|
89
|
+
"book": [
|
|
90
|
+
"book",
|
|
91
|
+
"bookData",
|
|
92
|
+
],
|
|
93
|
+
"adventure": [
|
|
94
|
+
"adventure",
|
|
95
|
+
"adventureData",
|
|
96
|
+
],
|
|
97
|
+
"makebrew": [
|
|
98
|
+
"makebrewCreatureTrait",
|
|
99
|
+
],
|
|
100
|
+
};
|
|
101
|
+
|
|
102
|
+
static _checkFileContents () {
|
|
103
|
+
Um.info(`PROP_CHECK`, `Checking file contents...`);
|
|
104
|
+
const results = [];
|
|
105
|
+
Uf.runOnDirs((dir) => {
|
|
106
|
+
if (dir === "collection") return;
|
|
107
|
+
|
|
108
|
+
Um.info(`PROP_CHECK`, `Checking dir "${dir}"...`);
|
|
109
|
+
const dirFiles = fs.readdirSync(dir, "utf8")
|
|
110
|
+
.filter(file => file.endsWith(".json"));
|
|
111
|
+
|
|
112
|
+
dirFiles.forEach(file => {
|
|
113
|
+
const json = JSON.parse(fs.readFileSync(`${dir}/${file}`, "utf-8"));
|
|
114
|
+
const props = this._DIR_TO_PRIMARY_PROP[dir] || [dir];
|
|
115
|
+
props.forEach(prop => {
|
|
116
|
+
if (!json[prop]) results.push(`${dir}/${file} was missing a "${prop}" property!`);
|
|
117
|
+
});
|
|
118
|
+
});
|
|
119
|
+
});
|
|
120
|
+
|
|
121
|
+
if (results.length) {
|
|
122
|
+
results.forEach(r => Um.error(`PROP_CHECK`, r));
|
|
123
|
+
throw new Error(`${results.length} file${results.length === 1 ? " was missing a primary prop!" : "s were missing primary props!"} See above for more info.`);
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
Um.info(`PROP_CHECK`, `Complete.`);
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
static _buildDeepIndex () {
|
|
130
|
+
const indexes = [
|
|
131
|
+
new _BrewIndexTimestamps(),
|
|
132
|
+
new _BrewIndexProps(),
|
|
133
|
+
new _BrewIndexSources(),
|
|
134
|
+
new _BrewIndexMeta(),
|
|
135
|
+
];
|
|
136
|
+
|
|
137
|
+
Um.info(`INDEX`, `Indexing...`);
|
|
138
|
+
|
|
139
|
+
Uf.runOnDirs((folder) => {
|
|
140
|
+
Um.info(`INDEX`, `Indexing dir "${folder}"...`);
|
|
141
|
+
|
|
142
|
+
Uf.listJsonFiles(folder)
|
|
143
|
+
.map(file => ({
|
|
144
|
+
folder,
|
|
145
|
+
name: file,
|
|
146
|
+
cleanName: file.replace(/#/g, "%23"),
|
|
147
|
+
contents: Uf.readJSON(file),
|
|
148
|
+
}))
|
|
149
|
+
.forEach(fileInfo => {
|
|
150
|
+
if (!fileInfo.contents._meta) {
|
|
151
|
+
throw new Error(`File "${fileInfo.name}" did not have metadata!`);
|
|
152
|
+
}
|
|
153
|
+
|
|
154
|
+
if (fileInfo.contents._meta.unlisted) return;
|
|
155
|
+
|
|
156
|
+
indexes.forEach(index => index.addToIndex(fileInfo));
|
|
157
|
+
});
|
|
158
|
+
});
|
|
159
|
+
|
|
160
|
+
fs.mkdirSync("_generated", {recursive: true});
|
|
161
|
+
|
|
162
|
+
indexes.forEach(index => index.doWrite());
|
|
163
|
+
}
|
|
164
|
+
|
|
165
|
+
static run () {
|
|
166
|
+
this._checkFileContents();
|
|
167
|
+
this._buildDeepIndex();
|
|
168
|
+
Um.info(`INDEX`, `Complete.`);
|
|
169
|
+
return null;
|
|
170
|
+
}
|
|
171
|
+
}
|
|
172
|
+
|
|
173
|
+
export {BrewIndexGenerator};
|
|
@@ -0,0 +1,102 @@
|
|
|
1
|
+
import {execFile} from "node:child_process";
|
|
2
|
+
import * as Uf from "./UtilFs.js";
|
|
3
|
+
import Um from "./UtilMisc.js";
|
|
4
|
+
import fs from "fs";
|
|
5
|
+
import hasha from "hasha";
|
|
6
|
+
import {getCleanJson} from "./UtilClean.js";
|
|
7
|
+
|
|
8
|
+
class BrewTimestamper {
|
|
9
|
+
static _LOG_TAG = `TIMESTAMPS`;
|
|
10
|
+
|
|
11
|
+
static _UPDATE_TYPES = {
|
|
12
|
+
NONE: 0,
|
|
13
|
+
HASH: 1,
|
|
14
|
+
TIMESTAMP: 2,
|
|
15
|
+
};
|
|
16
|
+
|
|
17
|
+
static async _pUpdateDir (dir) {
|
|
18
|
+
const promises = Uf.listJsonFiles(dir)
|
|
19
|
+
.map(async file => {
|
|
20
|
+
const fileData = Uf.readJSON(file, {isIncludeRaw: true});
|
|
21
|
+
|
|
22
|
+
if (!fileData.json._meta) {
|
|
23
|
+
throw new Error(`File "${file}" did not have metadata!`);
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
let updateType = this._UPDATE_TYPES.NONE;
|
|
27
|
+
|
|
28
|
+
// We hash the file without `dateAdded`, `dateLastModified`, and `_dateLastModifiedHash` to ensure the hash
|
|
29
|
+
// is stable when changing date values.
|
|
30
|
+
const toHashObj = {...fileData.json};
|
|
31
|
+
toHashObj._meta = {...toHashObj._meta, dateAdded: undefined, dateLastModified: undefined, _dateLastModifiedHash: undefined};
|
|
32
|
+
const expectedHash = (await hasha.async(JSON.stringify(toHashObj))).slice(0, 10);
|
|
33
|
+
|
|
34
|
+
if (!fileData.json._meta._dateLastModifiedHash) {
|
|
35
|
+
updateType = this._UPDATE_TYPES.HASH;
|
|
36
|
+
fileData.json._meta._dateLastModifiedHash = expectedHash;
|
|
37
|
+
} else if (expectedHash !== fileData.json._meta._dateLastModifiedHash) {
|
|
38
|
+
// Grab the last commit timestamp from the log.
|
|
39
|
+
// This is often a "junk" commit generated by cleaning (or indeed, timestamping) the file, but this is
|
|
40
|
+
// good enough.
|
|
41
|
+
const dateLastModified = await new Promise((resolve, reject) => {
|
|
42
|
+
execFile(
|
|
43
|
+
"git",
|
|
44
|
+
["log", "-1", `--format="%ad"`, file],
|
|
45
|
+
{
|
|
46
|
+
windowsHide: true,
|
|
47
|
+
},
|
|
48
|
+
(err, stdout, stderr) => {
|
|
49
|
+
if (err) return reject(err);
|
|
50
|
+
resolve(Math.round(new Date(stdout.trim()).getTime() / 1000));
|
|
51
|
+
},
|
|
52
|
+
);
|
|
53
|
+
});
|
|
54
|
+
|
|
55
|
+
if (fileData.json._meta.dateLastModified < dateLastModified) {
|
|
56
|
+
updateType = this._UPDATE_TYPES.TIMESTAMP;
|
|
57
|
+
fileData.json._meta.dateLastModified = dateLastModified;
|
|
58
|
+
fileData.json._meta._dateLastModifiedHash = expectedHash;
|
|
59
|
+
}
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
if (updateType === this._UPDATE_TYPES.NONE) return;
|
|
63
|
+
|
|
64
|
+
const strContents = getCleanJson(fileData.json);
|
|
65
|
+
|
|
66
|
+
await new Promise((resolve, reject) => {
|
|
67
|
+
fs.writeFile(
|
|
68
|
+
file,
|
|
69
|
+
strContents,
|
|
70
|
+
(err) => {
|
|
71
|
+
if (err) return reject(err);
|
|
72
|
+
resolve();
|
|
73
|
+
},
|
|
74
|
+
);
|
|
75
|
+
});
|
|
76
|
+
|
|
77
|
+
Um.info(
|
|
78
|
+
this._LOG_TAG,
|
|
79
|
+
updateType === this._UPDATE_TYPES.HASH
|
|
80
|
+
? `\t- Updated "_dateLastModifiedHash" for "${file}"...`
|
|
81
|
+
: `\t- Updated "dateLastModified" for "${file}"...`,
|
|
82
|
+
);
|
|
83
|
+
});
|
|
84
|
+
|
|
85
|
+
await Promise.all(promises);
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
static async pRun () {
|
|
89
|
+
await Uf.pRunOnDirs(
|
|
90
|
+
async (dir) => {
|
|
91
|
+
Um.info(this._LOG_TAG, `Updating dateLastModified timestamps in dir "${dir}"...`);
|
|
92
|
+
await this._pUpdateDir(dir);
|
|
93
|
+
},
|
|
94
|
+
{
|
|
95
|
+
isSerial: true,
|
|
96
|
+
},
|
|
97
|
+
);
|
|
98
|
+
Um.info(this._LOG_TAG, "Done!");
|
|
99
|
+
}
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
export {BrewTimestamper};
|
package/lib/TestJson.js
CHANGED
|
@@ -7,8 +7,8 @@ import Ajv2020 from "ajv/dist/2020.js";
|
|
|
7
7
|
import addFormats from "ajv-formats";
|
|
8
8
|
import * as jsonSourceMap from "json-source-map";
|
|
9
9
|
|
|
10
|
-
import * as
|
|
11
|
-
import
|
|
10
|
+
import * as Uf from "./UtilFs.js";
|
|
11
|
+
import Um from "./UtilMisc.js";
|
|
12
12
|
import {WorkerList, Deferred} from "./WorkerList.js";
|
|
13
13
|
|
|
14
14
|
const __dirname = url.fileURLToPath(new URL(".", import.meta.url));
|
|
@@ -23,12 +23,20 @@ class JsonTester {
|
|
|
23
23
|
|
|
24
24
|
constructor (
|
|
25
25
|
{
|
|
26
|
-
|
|
26
|
+
fnGetSchemaId,
|
|
27
|
+
|
|
28
|
+
isBrew = false,
|
|
29
|
+
dirSchema = null,
|
|
27
30
|
tagLog = LOG_TAG,
|
|
28
|
-
fnGetSchemaId = () => "homebrew.json",
|
|
29
31
|
},
|
|
30
32
|
) {
|
|
31
|
-
|
|
33
|
+
if (isBrew && dirSchema) throw new Error(`"isBrew" and "dirSchema" are mutually exclusive!`);
|
|
34
|
+
if (!fnGetSchemaId) throw new Error(`"fnGetSchemaId" is required!`);
|
|
35
|
+
|
|
36
|
+
this._dirSchema = dirSchema ??
|
|
37
|
+
(isBrew
|
|
38
|
+
? path.join(__dirname, "..", "schema", "brew")
|
|
39
|
+
: path.join(__dirname, "..", "schema", "site"));
|
|
32
40
|
this._tagLog = tagLog;
|
|
33
41
|
this._fnGetSchemaId = fnGetSchemaId;
|
|
34
42
|
|
|
@@ -98,7 +106,7 @@ class JsonTester {
|
|
|
98
106
|
});
|
|
99
107
|
|
|
100
108
|
const error = this._getFileErrors({filePath});
|
|
101
|
-
|
|
109
|
+
Um.error(this._tagLog, error);
|
|
102
110
|
out.errors.push(error);
|
|
103
111
|
|
|
104
112
|
return out;
|
|
@@ -107,10 +115,10 @@ class JsonTester {
|
|
|
107
115
|
_doLoadSchema () {
|
|
108
116
|
if (this._isSchemaLoaded) return;
|
|
109
117
|
|
|
110
|
-
|
|
118
|
+
Uf.listJsonFiles(this._dirSchema)
|
|
111
119
|
.forEach(filePath => {
|
|
112
120
|
filePath = path.normalize(filePath);
|
|
113
|
-
const contents =
|
|
121
|
+
const contents = Uf.readJSON(filePath);
|
|
114
122
|
|
|
115
123
|
const relativeFilePath = path.relative(this._dirSchema, filePath)
|
|
116
124
|
.replace(/\\/g, "/");
|
|
@@ -121,12 +129,17 @@ class JsonTester {
|
|
|
121
129
|
this._isSchemaLoaded = true;
|
|
122
130
|
}
|
|
123
131
|
|
|
132
|
+
_hasSchema (schemaId) {
|
|
133
|
+
this._doLoadSchema();
|
|
134
|
+
return !!this._ajv.schemas[schemaId];
|
|
135
|
+
}
|
|
136
|
+
|
|
124
137
|
getFileErrors ({filePath} = {}) {
|
|
125
138
|
this._doLoadSchema();
|
|
126
139
|
|
|
127
|
-
|
|
140
|
+
Um.info(this._tagLog, `\tValidating "${filePath}"...`);
|
|
128
141
|
|
|
129
|
-
const data =
|
|
142
|
+
const data = Uf.readJSON(filePath);
|
|
130
143
|
this._addImplicits(data);
|
|
131
144
|
|
|
132
145
|
const isValid = this._ajv.validate(this._fnGetSchemaId(filePath), data);
|
|
@@ -136,7 +149,7 @@ class JsonTester {
|
|
|
136
149
|
}
|
|
137
150
|
|
|
138
151
|
_doRunOnDir ({isFailFast, dir, errors, errorsFull, ...opts} = {}) {
|
|
139
|
-
for (const filePath of
|
|
152
|
+
for (const filePath of Uf.listJsonFiles(dir, opts)) {
|
|
140
153
|
const {errors: errorsFile, errorsFull: errorsFullFile} = this.getFileErrors({filePath});
|
|
141
154
|
errors.push(...errorsFile);
|
|
142
155
|
errorsFull.push(...errorsFullFile);
|
|
@@ -150,7 +163,7 @@ class JsonTester {
|
|
|
150
163
|
* @param [opts.dirBlocklist]
|
|
151
164
|
*/
|
|
152
165
|
getErrors (dir, opts = {}) {
|
|
153
|
-
|
|
166
|
+
Um.info(this._tagLog, `Validating JSON against schema`);
|
|
154
167
|
|
|
155
168
|
const errors = [];
|
|
156
169
|
const errorsFull = [];
|
|
@@ -161,12 +174,12 @@ class JsonTester {
|
|
|
161
174
|
}
|
|
162
175
|
|
|
163
176
|
getErrorsOnDirs ({isFailFast = false} = {}) {
|
|
164
|
-
|
|
177
|
+
Um.info(this._tagLog, `Validating JSON against schema`);
|
|
165
178
|
|
|
166
179
|
const errors = [];
|
|
167
180
|
const errorsFull = [];
|
|
168
181
|
|
|
169
|
-
|
|
182
|
+
Uf.runOnDirs((dir) => {
|
|
170
183
|
if (isFailFast && errors.length) return;
|
|
171
184
|
return this._doRunOnDir({isFailFast, errors, errorsFull, dir});
|
|
172
185
|
});
|
|
@@ -174,19 +187,30 @@ class JsonTester {
|
|
|
174
187
|
return {errors, errorsFull};
|
|
175
188
|
}
|
|
176
189
|
|
|
177
|
-
async pGetErrorsOnDirsWorkers ({isFailFast = false} = {}) {
|
|
178
|
-
|
|
190
|
+
async pGetErrorsOnDirsWorkers ({isFailFast = false, fileList = null} = {}) {
|
|
191
|
+
Um.info(this._tagLog, `Validating JSON against schema`);
|
|
179
192
|
|
|
180
193
|
const cntWorkers = Math.max(1, os.cpus().length - 1);
|
|
181
194
|
|
|
182
195
|
const errors = [];
|
|
183
196
|
const errorsFull = [];
|
|
184
197
|
|
|
185
|
-
const fileQueue = [];
|
|
198
|
+
const fileQueue = [...(fileList || [])];
|
|
186
199
|
|
|
187
|
-
|
|
188
|
-
|
|
200
|
+
if (!fileList) {
|
|
201
|
+
Uf.runOnDirs((dir) => {
|
|
202
|
+
fileQueue.push(...Uf.listJsonFiles(dir));
|
|
203
|
+
});
|
|
204
|
+
}
|
|
205
|
+
|
|
206
|
+
// region Verify that every file path maps to a valid schema
|
|
207
|
+
fileQueue.forEach(filePath => {
|
|
208
|
+
const schemaId = this._fnGetSchemaId(filePath);
|
|
209
|
+
if (!schemaId) throw new Error(`Failed to get schema ID for file path "${filePath}"`);
|
|
210
|
+
if (!this._hasSchema(schemaId)) throw new Error(`No schema loaded with schema ID "${schemaId}"`);
|
|
211
|
+
return schemaId;
|
|
189
212
|
});
|
|
213
|
+
// endregion
|
|
190
214
|
|
|
191
215
|
const workerList = new WorkerList();
|
|
192
216
|
|
package/lib/UtilClean.js
ADDED
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
const _CLEAN_JSON_REPLACEMENTS = {
|
|
2
|
+
"—": "\\u2014",
|
|
3
|
+
"–": "\\u2013",
|
|
4
|
+
"−": "\\u2212",
|
|
5
|
+
"“": `\\"`,
|
|
6
|
+
"”": `\\"`,
|
|
7
|
+
"’": "'",
|
|
8
|
+
"…": "...",
|
|
9
|
+
" ": " ", // non-breaking space
|
|
10
|
+
"ff": "ff",
|
|
11
|
+
"ffi": "ffi",
|
|
12
|
+
"ffl": "ffl",
|
|
13
|
+
"fi": "fi",
|
|
14
|
+
"fl": "fl",
|
|
15
|
+
"IJ": "IJ",
|
|
16
|
+
"ij": "ij",
|
|
17
|
+
"LJ": "LJ",
|
|
18
|
+
"Lj": "Lj",
|
|
19
|
+
"lj": "lj",
|
|
20
|
+
"NJ": "NJ",
|
|
21
|
+
"Nj": "Nj",
|
|
22
|
+
"nj": "nj",
|
|
23
|
+
"ſt": "ft",
|
|
24
|
+
};
|
|
25
|
+
const _CLEAN_JSON_REPLACEMENT_REGEX = new RegExp(Object.keys(_CLEAN_JSON_REPLACEMENTS).join("|"), "g");
|
|
26
|
+
|
|
27
|
+
const getCleanJson = (obj) => {
|
|
28
|
+
obj = `${JSON.stringify(obj, null, "\t")}\n`;
|
|
29
|
+
obj = obj.replace(_CLEAN_JSON_REPLACEMENT_REGEX, (match) => _CLEAN_JSON_REPLACEMENTS[match]);
|
|
30
|
+
return obj;
|
|
31
|
+
};
|
|
32
|
+
|
|
33
|
+
export {
|
|
34
|
+
getCleanJson,
|
|
35
|
+
};
|
package/lib/UtilMisc.js
CHANGED
|
@@ -1,22 +1,41 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
fn(`[${expandedTag}]`, ...args);
|
|
4
|
-
}
|
|
1
|
+
class MiscUtil {
|
|
2
|
+
/* -------------------------------------------- */
|
|
5
3
|
|
|
6
|
-
|
|
7
|
-
_taggedConsole(
|
|
8
|
-
|
|
4
|
+
// region Logging
|
|
5
|
+
static _taggedConsole (fn, tag, ...args) {
|
|
6
|
+
const expandedTag = tag.padStart(12, " ");
|
|
7
|
+
fn(`[${expandedTag}]`, ...args);
|
|
8
|
+
}
|
|
9
9
|
|
|
10
|
-
|
|
11
|
-
_taggedConsole(console.info, tag, ...args);
|
|
12
|
-
}
|
|
10
|
+
static warn (tag, ...args) { this._taggedConsole(console.warn, tag, ...args); }
|
|
11
|
+
static info (tag, ...args) { this._taggedConsole(console.info, tag, ...args); }
|
|
12
|
+
static error (tag, ...args) { this._taggedConsole(console.error, tag, ...args); }
|
|
13
|
+
// endregion
|
|
14
|
+
|
|
15
|
+
/* -------------------------------------------- */
|
|
16
|
+
|
|
17
|
+
static get (object, ...path) {
|
|
18
|
+
if (object == null) return null;
|
|
19
|
+
for (let i = 0; i < path.length; ++i) {
|
|
20
|
+
object = object[path[i]];
|
|
21
|
+
if (object == null) return object;
|
|
22
|
+
}
|
|
23
|
+
return object;
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
/* -------------------------------------------- */
|
|
27
|
+
|
|
28
|
+
static copyFast (obj) {
|
|
29
|
+
if ((typeof obj !== "object") || obj == null) return obj;
|
|
30
|
+
|
|
31
|
+
if (obj instanceof Array) return obj.map(MiscUtil.copyFast);
|
|
32
|
+
|
|
33
|
+
const cpy = {};
|
|
34
|
+
for (const k of Object.keys(obj)) cpy[k] = MiscUtil.copyFast(obj[k]);
|
|
35
|
+
return cpy;
|
|
36
|
+
}
|
|
13
37
|
|
|
14
|
-
|
|
15
|
-
_taggedConsole(console.error, tag, ...args);
|
|
38
|
+
/* -------------------------------------------- */
|
|
16
39
|
}
|
|
17
40
|
|
|
18
|
-
export
|
|
19
|
-
warn,
|
|
20
|
-
info,
|
|
21
|
-
error,
|
|
22
|
-
};
|
|
41
|
+
export default MiscUtil;
|