@heroiclands/package-build 23.4.0 → 23.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +34 -0
- package/README.md +29 -25
- package/assets/images/provenance.yaml +1 -0
- package/bin/package-build.mjs +259 -28
- package/ci/prettier.mjs +73 -0
- package/config.mjs +10 -0
- package/container.mjs +232 -1
- package/content-config.mjs +5 -1
- package/docs/api.md +100 -52
- package/docs/authoring/assets.md +16 -1
- package/docs/commands.md +154 -7
- package/docs/configuration.md +39 -2
- package/docs/content-format.md +30 -6
- package/docs/guides/e2e.md +38 -0
- package/docs/guides/icons.md +5 -0
- package/docs/guides/release-and-deploy.md +36 -1
- package/docs/reference/format-details.md +218 -50
- package/docs/reference/note-types.md +71 -71
- package/engine/asset-index.mjs +63 -22
- package/engine/asset-references.mjs +355 -0
- package/engine/asset-shipping.mjs +363 -0
- package/engine/asset-types.mjs +70 -0
- package/engine/base-compiler.mjs +24 -1
- package/engine/cdn-publish.mjs +618 -0
- package/engine/index.mjs +6 -0
- package/engine/note-vocabulary.mjs +40 -2
- package/engine/pathnames.mjs +21 -10
- package/engine/rank-ladder.mjs +40 -16
- package/engine/scenes.mjs +1 -1
- package/engine/standing-terms.mjs +3 -1
- package/package.json +4 -3
- package/packagebuild-metadata.jsonl +13 -13
- package/release.mjs +69 -7
- package/sohl/actors.mjs +12 -4
- package/sohl/default-item-art.mjs +23 -19
- package/types/config.d.mts +7 -0
- package/types/container.d.mts +137 -1
- package/types/engine/asset-index.d.mts +3 -2
- package/types/engine/asset-references.d.mts +136 -0
- package/types/engine/asset-shipping.d.mts +240 -0
- package/types/engine/asset-types.d.mts +36 -0
- package/types/engine/base-compiler.d.mts +7 -1
- package/types/engine/cdn-publish.d.mts +311 -0
- package/types/engine/index.d.mts +2 -0
- package/types/engine/pathnames.d.mts +2 -1
- package/types/engine/rank-ladder.d.mts +25 -6
- package/types/release.d.mts +5 -3
- package/types/sohl/default-item-art.d.mts +2 -1
|
@@ -0,0 +1,618 @@
|
|
|
1
|
+
/*
|
|
2
|
+
* This file is part of the Song of Heroic Lands (SoHL) system for Foundry VTT.
|
|
3
|
+
* Copyright (c) 2024-2026 Tom Rodriguez ("Toasty") — <toasty@heroiclands.org>
|
|
4
|
+
*
|
|
5
|
+
* This work is licensed under the GNU General Public License v3.0 (GPLv3).
|
|
6
|
+
* You may copy, modify, and distribute it under the terms of that license.
|
|
7
|
+
*
|
|
8
|
+
* For full terms, see the LICENSE.md file in the project root or visit:
|
|
9
|
+
* https://www.gnu.org/licenses/gpl-3.0.html
|
|
10
|
+
*
|
|
11
|
+
* SPDX-License-Identifier: GPL-3.0-or-later
|
|
12
|
+
*/
|
|
13
|
+
|
|
14
|
+
/**
|
|
15
|
+
* `package-build cdn publish` — every picture a package has, on the CDN, and
|
|
16
|
+
* the art the package leaves out as a named asset set.
|
|
17
|
+
*
|
|
18
|
+
* **The layout is the deployed one.** Each icon and image is uploaded flat, as
|
|
19
|
+
* `icons/<basename>` or `images/<basename>`, under `<site.assets>/<package>/`
|
|
20
|
+
* — the address {@link module:engine/pathnames.resolvePathname} gives the
|
|
21
|
+
* website for the same file, so the site and the CDN agree by construction.
|
|
22
|
+
* Shipped and excluded pictures both go up: the website shows all of them.
|
|
23
|
+
*
|
|
24
|
+
* **The bucket is served at `site.assets`.** An object's key is its path below
|
|
25
|
+
* that URL, `<package>/images/<basename>`, so the key and the public address
|
|
26
|
+
* differ only by the host.
|
|
27
|
+
*
|
|
28
|
+
* **The bytes are the staged bytes.** The repository's `assetTransform` runs
|
|
29
|
+
* on each file exactly as the stage runs it, so a picture on the CDN and the
|
|
30
|
+
* same picture in the Foundry package are one file.
|
|
31
|
+
*
|
|
32
|
+
* **The set lists only what the package leaves out.** `--set <file>` names it,
|
|
33
|
+
* `<package>/<file>` on the CDN, in the shape
|
|
34
|
+
* {@link module:engine/asset-shipping} describes, with each `source` the
|
|
35
|
+
* file's absolute CDN URL. An address the set does not list resolves to the
|
|
36
|
+
* shipped copy, so listing a shipped picture would only make a world download
|
|
37
|
+
* a file it already has. `default.json` is refused: it is the shipped set's
|
|
38
|
+
* name.
|
|
39
|
+
*
|
|
40
|
+
* **A second run uploads nothing.** A picture whose object already exists at
|
|
41
|
+
* the same size is skipped. The set file is compared byte for byte, since a
|
|
42
|
+
* changed set can keep its length.
|
|
43
|
+
*
|
|
44
|
+
* **A file is refused; a document is reported.** A file under a picture root
|
|
45
|
+
* whose name is no address stops the run before anything is uploaded: the
|
|
46
|
+
* CDN has no key to put it under. A note naming a picture the package does
|
|
47
|
+
* not have, or naming left-out art the CDN does not hold once the run ends, is
|
|
48
|
+
* a warning against the note — the compile draws its default art for the
|
|
49
|
+
* first and the stand-in for the second, so neither stops a publish.
|
|
50
|
+
*
|
|
51
|
+
* Nothing here reads the environment or writes to the console: the command
|
|
52
|
+
* line supplies the credentials, the store and the log.
|
|
53
|
+
*
|
|
54
|
+
* @module
|
|
55
|
+
*/
|
|
56
|
+
|
|
57
|
+
import fs from "node:fs";
|
|
58
|
+
import path from "node:path";
|
|
59
|
+
|
|
60
|
+
import { AwsClient } from "aws4fetch";
|
|
61
|
+
|
|
62
|
+
import { artTarget, ART_SLOTS } from "./art-fields.mjs";
|
|
63
|
+
import { collectAssetRecords } from "./asset-index.mjs";
|
|
64
|
+
import { deployedAssetPath, selectStagedAssets, SHIPPED_SET_FILE } from "./asset-shipping.mjs";
|
|
65
|
+
import {
|
|
66
|
+
ASSET_SYSTEM,
|
|
67
|
+
ASSET_TYPE_NAMES,
|
|
68
|
+
DEPLOYED_ASSET_TYPES,
|
|
69
|
+
isAssetType,
|
|
70
|
+
} from "./asset-types.mjs";
|
|
71
|
+
import { renderAddress } from "./address.mjs";
|
|
72
|
+
import { EMBED_DEFAULT_TYPE, embedsIn } from "./content-embeds.mjs";
|
|
73
|
+
import { imagesIn } from "./content-images.mjs";
|
|
74
|
+
import { positionOfFrontmatterPath } from "./diagnostics.mjs";
|
|
75
|
+
import { walkMarkdownTree } from "./helpers.mjs";
|
|
76
|
+
import { resolvePathname } from "./pathnames.mjs";
|
|
77
|
+
|
|
78
|
+
/**
|
|
79
|
+
* The environment variables naming the S3-compatible store behind the CDN.
|
|
80
|
+
*
|
|
81
|
+
* For Cloudflare R2, `CDN_ENDPOINT` is the account's S3 endpoint,
|
|
82
|
+
* `https://<account id>.r2.cloudflarestorage.com`, and the key pair is an R2
|
|
83
|
+
* API token's.
|
|
84
|
+
*
|
|
85
|
+
* @type {Readonly<{endpoint: string, bucket: string, accessKeyId: string, secretAccessKey: string}>}
|
|
86
|
+
*/
|
|
87
|
+
export const CDN_ENV = Object.freeze({
|
|
88
|
+
endpoint: "CDN_ENDPOINT",
|
|
89
|
+
bucket: "CDN_BUCKET",
|
|
90
|
+
accessKeyId: "CDN_ACCESS_KEY_ID",
|
|
91
|
+
secretAccessKey: "CDN_SECRET_ACCESS_KEY",
|
|
92
|
+
});
|
|
93
|
+
|
|
94
|
+
/** How many requests run against the store at once. */
|
|
95
|
+
const CONCURRENCY = 8;
|
|
96
|
+
|
|
97
|
+
/**
|
|
98
|
+
* The media type each picture extension is served with. A browser refuses to
|
|
99
|
+
* draw an SVG served as anything but `image/svg+xml`.
|
|
100
|
+
*
|
|
101
|
+
* @type {Readonly<Record<string, string>>}
|
|
102
|
+
*/
|
|
103
|
+
const CONTENT_TYPES = Object.freeze({
|
|
104
|
+
".apng": "image/apng",
|
|
105
|
+
".avif": "image/avif",
|
|
106
|
+
".bmp": "image/bmp",
|
|
107
|
+
".gif": "image/gif",
|
|
108
|
+
".jpeg": "image/jpeg",
|
|
109
|
+
".jpg": "image/jpeg",
|
|
110
|
+
".json": "application/json",
|
|
111
|
+
".png": "image/png",
|
|
112
|
+
".svg": "image/svg+xml",
|
|
113
|
+
".tiff": "image/tiff",
|
|
114
|
+
".webp": "image/webp",
|
|
115
|
+
});
|
|
116
|
+
|
|
117
|
+
/**
|
|
118
|
+
* The media type a file is uploaded with.
|
|
119
|
+
*
|
|
120
|
+
* @param {string} name - A file name or key.
|
|
121
|
+
* @returns {string} Its media type, `application/octet-stream` when unknown.
|
|
122
|
+
*/
|
|
123
|
+
export function contentTypeOf(name) {
|
|
124
|
+
return CONTENT_TYPES[path.extname(name).toLowerCase()] ?? "application/octet-stream";
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
/**
|
|
128
|
+
* Why a `--set` value cannot name a set file, or `""` when it can.
|
|
129
|
+
*
|
|
130
|
+
* @param {unknown} file - The value given.
|
|
131
|
+
* @returns {string} The problem, as a clause, or `""`.
|
|
132
|
+
*/
|
|
133
|
+
export function setFileProblem(file) {
|
|
134
|
+
const name = String(file ?? "");
|
|
135
|
+
if (!name) return "`--set` names no file";
|
|
136
|
+
if (name.includes("/") || name.includes("\\")) {
|
|
137
|
+
return `\`--set ${name}\` is a path; a set file sits directly under the package's CDN root, so name a file`;
|
|
138
|
+
}
|
|
139
|
+
if (!name.endsWith(".json")) return `\`--set ${name}\` is not a \`.json\` file`;
|
|
140
|
+
if (name === path.posix.basename(SHIPPED_SET_FILE)) {
|
|
141
|
+
return (
|
|
142
|
+
`\`--set ${name}\` is refused: \`${name}\` is the shipped set's name, the ` +
|
|
143
|
+
"fallback every other set is read against. Name the set for its art"
|
|
144
|
+
);
|
|
145
|
+
}
|
|
146
|
+
return "";
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
/**
|
|
150
|
+
* The store's settings from the environment.
|
|
151
|
+
*
|
|
152
|
+
* @param {Record<string, string|undefined>} env - The environment.
|
|
153
|
+
* @returns {{endpoint: string, bucket: string, accessKeyId: string,
|
|
154
|
+
* secretAccessKey: string}|null} The settings, or `null` when none of the
|
|
155
|
+
* variables is set.
|
|
156
|
+
* @throws {Error} When some are set and others are not, naming the missing
|
|
157
|
+
* ones and never a value.
|
|
158
|
+
*/
|
|
159
|
+
export function cdnCredentials(env) {
|
|
160
|
+
const values = Object.fromEntries(
|
|
161
|
+
Object.entries(CDN_ENV).map(([key, name]) => [key, String(env[name] ?? "").trim()]),
|
|
162
|
+
);
|
|
163
|
+
const missing = Object.entries(CDN_ENV)
|
|
164
|
+
.filter(([key]) => !values[key])
|
|
165
|
+
.map(([, name]) => name);
|
|
166
|
+
if (missing.length === Object.keys(CDN_ENV).length) return null;
|
|
167
|
+
if (missing.length) {
|
|
168
|
+
throw new Error(
|
|
169
|
+
`the CDN store is half configured: ${missing.join(", ")} ` +
|
|
170
|
+
`${missing.length === 1 ? "is" : "are"} not set. Set every one of ` +
|
|
171
|
+
`${Object.values(CDN_ENV).join(", ")}, in the environment or \`.env.local\``,
|
|
172
|
+
);
|
|
173
|
+
}
|
|
174
|
+
return /** @type {any} */ (values);
|
|
175
|
+
}
|
|
176
|
+
|
|
177
|
+
/**
|
|
178
|
+
* A store holding the CDN's objects.
|
|
179
|
+
*
|
|
180
|
+
* @typedef {object} CdnStore
|
|
181
|
+
* @property {(key: string) => Promise<number|undefined>} size - An object's
|
|
182
|
+
* size in bytes, `undefined` when there is none.
|
|
183
|
+
* @property {(key: string) => Promise<Buffer|undefined>} read - An object's
|
|
184
|
+
* bytes, `undefined` when there is none.
|
|
185
|
+
* @property {((key: string, body: Buffer, contentType: string) => Promise<void>)|undefined} write -
|
|
186
|
+
* Store an object; absent on a read-only store.
|
|
187
|
+
* @property {string} describe - What the store is, for the log. Never a
|
|
188
|
+
* secret.
|
|
189
|
+
*/
|
|
190
|
+
|
|
191
|
+
/**
|
|
192
|
+
* Throw for a response the store refused.
|
|
193
|
+
*
|
|
194
|
+
* @param {string} what - The request, as `METHOD key`.
|
|
195
|
+
* @param {Response} response - The response.
|
|
196
|
+
* @returns {never}
|
|
197
|
+
*/
|
|
198
|
+
function refused(what, response) {
|
|
199
|
+
throw new Error(`${what} answered ${response.status} ${response.statusText}`.trim());
|
|
200
|
+
}
|
|
201
|
+
|
|
202
|
+
/**
|
|
203
|
+
* The S3-compatible store the CDN is served from, signed with AWS SigV4.
|
|
204
|
+
*
|
|
205
|
+
* Addressed path-style, `<endpoint>/<bucket>/<key>`, which R2 and every other
|
|
206
|
+
* S3-compatible server accept.
|
|
207
|
+
*
|
|
208
|
+
* @param {object} opts
|
|
209
|
+
* @param {string} opts.endpoint - The S3 endpoint.
|
|
210
|
+
* @param {string} opts.bucket - The bucket.
|
|
211
|
+
* @param {string} opts.accessKeyId - The access key id.
|
|
212
|
+
* @param {string} opts.secretAccessKey - The secret access key.
|
|
213
|
+
* @param {typeof fetch} [opts.fetch] - The fetch to sign requests for.
|
|
214
|
+
* @returns {CdnStore} The store.
|
|
215
|
+
*/
|
|
216
|
+
export function s3Store({ endpoint, bucket, accessKeyId, secretAccessKey, fetch: fetchImpl }) {
|
|
217
|
+
const client = new AwsClient({ accessKeyId, secretAccessKey, service: "s3", region: "auto" });
|
|
218
|
+
const base = `${endpoint.replace(/\/+$/, "")}/${encodeURIComponent(bucket)}`;
|
|
219
|
+
const url = (key) => `${base}/${key.split("/").map(encodeURIComponent).join("/")}`;
|
|
220
|
+
const send = async (key, init) => {
|
|
221
|
+
const request = await client.sign(url(key), init);
|
|
222
|
+
return (fetchImpl ?? fetch)(request);
|
|
223
|
+
};
|
|
224
|
+
return {
|
|
225
|
+
describe: `bucket ${bucket} at ${new URL(endpoint).host}`,
|
|
226
|
+
async size(key) {
|
|
227
|
+
const response = await send(key, { method: "HEAD" });
|
|
228
|
+
if (response.status === 404) return undefined;
|
|
229
|
+
if (!response.ok) refused(`HEAD ${key}`, response);
|
|
230
|
+
return Number(response.headers.get("content-length"));
|
|
231
|
+
},
|
|
232
|
+
async read(key) {
|
|
233
|
+
const response = await send(key, { method: "GET" });
|
|
234
|
+
if (response.status === 404) return undefined;
|
|
235
|
+
if (!response.ok) refused(`GET ${key}`, response);
|
|
236
|
+
return Buffer.from(await response.arrayBuffer());
|
|
237
|
+
},
|
|
238
|
+
async write(key, body, contentType) {
|
|
239
|
+
const response = await send(key, {
|
|
240
|
+
method: "PUT",
|
|
241
|
+
body,
|
|
242
|
+
headers: { "content-type": contentType, "content-length": String(body.length) },
|
|
243
|
+
});
|
|
244
|
+
if (!response.ok) refused(`PUT ${key}`, response);
|
|
245
|
+
},
|
|
246
|
+
};
|
|
247
|
+
}
|
|
248
|
+
|
|
249
|
+
/**
|
|
250
|
+
* The CDN read through its public address — what a dry run without
|
|
251
|
+
* credentials compares against. It cannot write.
|
|
252
|
+
*
|
|
253
|
+
* @param {string} host - `site.assets`, the URL the bucket is served at.
|
|
254
|
+
* @param {typeof fetch} [fetchImpl] - The fetch to use.
|
|
255
|
+
* @returns {CdnStore} The read-only store.
|
|
256
|
+
*/
|
|
257
|
+
export function publicStore(host, fetchImpl) {
|
|
258
|
+
const base = host.replace(/\/+$/, "");
|
|
259
|
+
const get = (key, method) => (fetchImpl ?? fetch)(`${base}/${key}`, { method });
|
|
260
|
+
return {
|
|
261
|
+
describe: `${base}, read-only`,
|
|
262
|
+
async size(key) {
|
|
263
|
+
const response = await get(key, "HEAD");
|
|
264
|
+
if (response.status === 404 || response.status === 403) return undefined;
|
|
265
|
+
if (!response.ok) refused(`HEAD ${key}`, response);
|
|
266
|
+
return Number(response.headers.get("content-length"));
|
|
267
|
+
},
|
|
268
|
+
async read(key) {
|
|
269
|
+
const response = await get(key, "GET");
|
|
270
|
+
if (response.status === 404 || response.status === 403) return undefined;
|
|
271
|
+
if (!response.ok) refused(`GET ${key}`, response);
|
|
272
|
+
return Buffer.from(await response.arrayBuffer());
|
|
273
|
+
},
|
|
274
|
+
write: undefined,
|
|
275
|
+
};
|
|
276
|
+
}
|
|
277
|
+
|
|
278
|
+
/**
|
|
279
|
+
* One picture to publish.
|
|
280
|
+
*
|
|
281
|
+
* @typedef {object} CdnUpload
|
|
282
|
+
* @property {object} record - The asset record.
|
|
283
|
+
* @property {string} address - Its canonical address.
|
|
284
|
+
* @property {string} target - `<icons|images>/<basename>`.
|
|
285
|
+
* @property {string} key - The object key, `<package>/<target>`.
|
|
286
|
+
* @property {Buffer} body - The bytes, transformed as the stage transforms
|
|
287
|
+
* them.
|
|
288
|
+
* @property {string} contentType - The media type it is served with.
|
|
289
|
+
* @property {boolean} shipped - Whether the Foundry package ships it.
|
|
290
|
+
*/
|
|
291
|
+
|
|
292
|
+
/**
|
|
293
|
+
* Everything a publish uploads, worked out before anything is sent.
|
|
294
|
+
*
|
|
295
|
+
* @typedef {object} CdnPlan
|
|
296
|
+
* @property {object[]} findings - What stops the publish; nothing is planned
|
|
297
|
+
* when non-empty.
|
|
298
|
+
* @property {object[]} records - Every asset record the package has.
|
|
299
|
+
* @property {CdnUpload[]} uploads - One per picture, in key order.
|
|
300
|
+
* @property {{key: string, body: Buffer, set: object}|null} set - The set file.
|
|
301
|
+
* @property {string} cdnRoot - `<site.assets>/<package>/`.
|
|
302
|
+
*/
|
|
303
|
+
|
|
304
|
+
/**
|
|
305
|
+
* Work out what a publish uploads.
|
|
306
|
+
*
|
|
307
|
+
* @param {object} opts
|
|
308
|
+
* @param {string} opts.assetsBase - The package's asset directory.
|
|
309
|
+
* @param {string} opts.contentPackage - The content package.
|
|
310
|
+
* @param {string} opts.host - `site.assets`.
|
|
311
|
+
* @param {string} opts.setFile - The set file's name.
|
|
312
|
+
* @param {string} opts.setName - The set's display name.
|
|
313
|
+
* @param {(sourcePath: string) => string|null|undefined} [opts.transform] -
|
|
314
|
+
* The repository's per-file transform.
|
|
315
|
+
* @param {readonly string[]} [opts.shippingLicenses] - The licences the
|
|
316
|
+
* package adds to the shipping list, so the set leaves out what the stage
|
|
317
|
+
* leaves out.
|
|
318
|
+
* @returns {CdnPlan} The plan.
|
|
319
|
+
*/
|
|
320
|
+
export function planCdnPublish({
|
|
321
|
+
assetsBase,
|
|
322
|
+
contentPackage,
|
|
323
|
+
host,
|
|
324
|
+
setFile,
|
|
325
|
+
setName,
|
|
326
|
+
transform,
|
|
327
|
+
shippingLicenses = [],
|
|
328
|
+
}) {
|
|
329
|
+
const findings = [];
|
|
330
|
+
const cdnRoot = `${host.replace(/\/+$/, "")}/${contentPackage}/`;
|
|
331
|
+
const records = collectAssetRecords(assetsBase, { contentPackage, problems: findings });
|
|
332
|
+
if (findings.length) return { findings, records, uploads: [], set: null, cdnRoot };
|
|
333
|
+
|
|
334
|
+
const { staged, excluded } = selectStagedAssets(records, shippingLicenses);
|
|
335
|
+
const shipped = new Set(staged);
|
|
336
|
+
const uploads = [...staged, ...excluded.map(({ record }) => record)]
|
|
337
|
+
.map((record) => {
|
|
338
|
+
const from = path.join(assetsBase, ...record.asset.path.split("/"));
|
|
339
|
+
const transformed = transform ? transform(from) : null;
|
|
340
|
+
const body =
|
|
341
|
+
transformed != null ? Buffer.from(transformed, "utf8") : fs.readFileSync(from);
|
|
342
|
+
const target = /** @type {string} */ (deployedAssetPath(record));
|
|
343
|
+
return {
|
|
344
|
+
record,
|
|
345
|
+
address: record.address.canonical,
|
|
346
|
+
target,
|
|
347
|
+
key: `${contentPackage}/${target}`,
|
|
348
|
+
body,
|
|
349
|
+
contentType: contentTypeOf(target),
|
|
350
|
+
shipped: shipped.has(record),
|
|
351
|
+
};
|
|
352
|
+
})
|
|
353
|
+
.sort((a, b) => (a.key < b.key ? -1 : 1));
|
|
354
|
+
|
|
355
|
+
const set = {
|
|
356
|
+
name: setName,
|
|
357
|
+
package: contentPackage,
|
|
358
|
+
assets: uploads
|
|
359
|
+
.filter((upload) => !upload.shipped)
|
|
360
|
+
.map((upload) => ({
|
|
361
|
+
address: upload.address,
|
|
362
|
+
source: `${cdnRoot}${upload.target}`,
|
|
363
|
+
target: upload.target,
|
|
364
|
+
size: upload.body.length,
|
|
365
|
+
}))
|
|
366
|
+
.sort((a, b) => (a.target < b.target ? -1 : 1)),
|
|
367
|
+
};
|
|
368
|
+
return {
|
|
369
|
+
findings,
|
|
370
|
+
records,
|
|
371
|
+
uploads,
|
|
372
|
+
set: {
|
|
373
|
+
key: `${contentPackage}/${setFile}`,
|
|
374
|
+
body: Buffer.from(`${JSON.stringify(set, null, 4)}\n`, "utf8"),
|
|
375
|
+
set,
|
|
376
|
+
},
|
|
377
|
+
cdnRoot,
|
|
378
|
+
};
|
|
379
|
+
}
|
|
380
|
+
|
|
381
|
+
/**
|
|
382
|
+
* Run `work` over `items`, `limit` at a time, in order of completion.
|
|
383
|
+
*
|
|
384
|
+
* @template T
|
|
385
|
+
* @param {readonly T[]} items
|
|
386
|
+
* @param {number} limit
|
|
387
|
+
* @param {(item: T) => Promise<void>} work
|
|
388
|
+
* @returns {Promise<void>}
|
|
389
|
+
*/
|
|
390
|
+
async function pooled(items, limit, work) {
|
|
391
|
+
let next = 0;
|
|
392
|
+
const runners = Array.from({ length: Math.min(limit, items.length) }, async () => {
|
|
393
|
+
while (next < items.length) {
|
|
394
|
+
const item = items[next++];
|
|
395
|
+
await work(item);
|
|
396
|
+
}
|
|
397
|
+
});
|
|
398
|
+
await Promise.all(runners);
|
|
399
|
+
}
|
|
400
|
+
|
|
401
|
+
/**
|
|
402
|
+
* Upload a plan.
|
|
403
|
+
*
|
|
404
|
+
* A picture that fails is recorded and the rest carry on, so one refused file
|
|
405
|
+
* costs one file rather than the run. The set file is written only when every
|
|
406
|
+
* picture it lists is on the CDN: a set naming a file the CDN lacks sends a
|
|
407
|
+
* world to fetch nothing.
|
|
408
|
+
*
|
|
409
|
+
* @param {CdnPlan} plan - From {@link planCdnPublish}, with no findings.
|
|
410
|
+
* @param {object} opts
|
|
411
|
+
* @param {CdnStore} opts.store - Where the objects go.
|
|
412
|
+
* @param {boolean} [opts.dryRun] - Compare only; write nothing.
|
|
413
|
+
* @param {(line: string) => void} [opts.log] - One line per object.
|
|
414
|
+
* @returns {Promise<{uploaded: CdnUpload[], skipped: CdnUpload[],
|
|
415
|
+
* failed: {upload: CdnUpload, message: string}[],
|
|
416
|
+
* set: "uploaded"|"skipped"|"withheld", published: Set<string>}>} What was
|
|
417
|
+
* (or, in a dry run, would be) uploaded, skipped and refused; what became of
|
|
418
|
+
* the set file; and every address whose picture is on the CDN at its
|
|
419
|
+
* planned size once the run ends — or would be, in a dry run.
|
|
420
|
+
*/
|
|
421
|
+
export async function publishToCdn(plan, { store, dryRun = false, log = () => {} }) {
|
|
422
|
+
if (!dryRun && !store.write) {
|
|
423
|
+
throw new Error(`the store (${store.describe}) is read-only, so it can only be dry-run`);
|
|
424
|
+
}
|
|
425
|
+
const write = /** @type {NonNullable<CdnStore["write"]>} */ (store.write);
|
|
426
|
+
const verb = dryRun ? "would upload" : "uploaded";
|
|
427
|
+
const uploaded = [];
|
|
428
|
+
const skipped = [];
|
|
429
|
+
const failed = [];
|
|
430
|
+
const published = new Set();
|
|
431
|
+
await pooled(plan.uploads, CONCURRENCY, async (upload) => {
|
|
432
|
+
try {
|
|
433
|
+
const size = await store.size(upload.key);
|
|
434
|
+
if (size === upload.body.length) {
|
|
435
|
+
skipped.push(upload);
|
|
436
|
+
} else {
|
|
437
|
+
if (!dryRun) await write(upload.key, upload.body, upload.contentType);
|
|
438
|
+
uploaded.push(upload);
|
|
439
|
+
}
|
|
440
|
+
published.add(upload.address);
|
|
441
|
+
} catch (err) {
|
|
442
|
+
failed.push({ upload, message: err instanceof Error ? err.message : String(err) });
|
|
443
|
+
}
|
|
444
|
+
});
|
|
445
|
+
const byKey = (a, b) => (a.key < b.key ? -1 : 1);
|
|
446
|
+
uploaded.sort(byKey);
|
|
447
|
+
skipped.sort(byKey);
|
|
448
|
+
failed.sort((a, b) => byKey(a.upload, b.upload));
|
|
449
|
+
for (const upload of uploaded) log(`${verb} ${upload.key} (${upload.body.length} bytes)`);
|
|
450
|
+
for (const upload of skipped) log(`skipped ${upload.key}, already present at that size`);
|
|
451
|
+
|
|
452
|
+
const setPlan = /** @type {NonNullable<CdnPlan["set"]>} */ (plan.set);
|
|
453
|
+
if (failed.length) {
|
|
454
|
+
log(`withheld ${setPlan.key}: ${failed.length} of its pictures did not upload`);
|
|
455
|
+
return { uploaded, skipped, failed, set: "withheld", published };
|
|
456
|
+
}
|
|
457
|
+
const current = await store.read(setPlan.key);
|
|
458
|
+
if (current && current.equals(setPlan.body)) {
|
|
459
|
+
log(`skipped ${setPlan.key}, unchanged`);
|
|
460
|
+
return { uploaded, skipped, failed, set: "skipped", published };
|
|
461
|
+
}
|
|
462
|
+
if (!dryRun) await write(setPlan.key, setPlan.body, contentTypeOf(setPlan.key));
|
|
463
|
+
log(`${verb} ${setPlan.key} (${setPlan.set.assets.length} assets)`);
|
|
464
|
+
return { uploaded, skipped, failed, set: "uploaded", published };
|
|
465
|
+
}
|
|
466
|
+
|
|
467
|
+
/**
|
|
468
|
+
* Line and column, 1-based, of an offset inside a note's body.
|
|
469
|
+
*
|
|
470
|
+
* @param {string} body - The body as the walk yields it.
|
|
471
|
+
* @param {number} offset - Where in the body.
|
|
472
|
+
* @param {number} bodyLine - The file line the body starts on.
|
|
473
|
+
* @param {number} bodyColumn - The file column it starts at.
|
|
474
|
+
* @returns {{line: number, column: number}} The file position.
|
|
475
|
+
*/
|
|
476
|
+
function bodyPosition(body, offset, bodyLine, bodyColumn) {
|
|
477
|
+
const before = body.slice(0, offset);
|
|
478
|
+
const lines = before.split("\n");
|
|
479
|
+
return lines.length === 1 ?
|
|
480
|
+
{ line: bodyLine, column: bodyColumn + offset }
|
|
481
|
+
: { line: bodyLine + lines.length - 1, column: lines[lines.length - 1].length + 1 };
|
|
482
|
+
}
|
|
483
|
+
|
|
484
|
+
/**
|
|
485
|
+
* Every place a package's notes name one of its own pictures, and what each
|
|
486
|
+
* names.
|
|
487
|
+
*
|
|
488
|
+
* Reads the three places a note names art: an art slot under `data:`, an
|
|
489
|
+
* embed (`![[address|label]]`) and a body image whose pathname is the
|
|
490
|
+
* package's own. A reference naming another package, or one that does not
|
|
491
|
+
* parse as an address, is another check's business and is not returned.
|
|
492
|
+
*
|
|
493
|
+
* @param {object} opts
|
|
494
|
+
* @param {string} opts.contentBase - The content tree.
|
|
495
|
+
* @param {readonly string[]} opts.skipDirectories - Directory names the walk
|
|
496
|
+
* ignores.
|
|
497
|
+
* @param {string} opts.contentPackage - The content package.
|
|
498
|
+
* @param {readonly object[]} opts.records - The package's asset records.
|
|
499
|
+
* @returns {Array<{file: string, line?: number, column?: number,
|
|
500
|
+
* written: string, record: object|null}>} One per reference, `record` `null`
|
|
501
|
+
* where the package has no such picture. `file` is relative to the working
|
|
502
|
+
* directory.
|
|
503
|
+
*/
|
|
504
|
+
export function noteArtReferences({ contentBase, skipDirectories, contentPackage, records }) {
|
|
505
|
+
const byAddress = new Map(records.map((record) => [record.address.canonical, record]));
|
|
506
|
+
const byPath = new Map(records.map((record) => [record.asset?.path, record]));
|
|
507
|
+
const vocabulary = {
|
|
508
|
+
system: ASSET_SYSTEM,
|
|
509
|
+
package: contentPackage,
|
|
510
|
+
types: ASSET_TYPE_NAMES,
|
|
511
|
+
packages: new Set([contentPackage]),
|
|
512
|
+
};
|
|
513
|
+
/** @returns {string|undefined} The canonical address, when it is this package's. */
|
|
514
|
+
const ownAddress = (value, defaultType, accepts) => {
|
|
515
|
+
const tuple = artTarget(value, defaultType, accepts, vocabulary);
|
|
516
|
+
if (tuple.reason || tuple.package !== contentPackage || !isAssetType(tuple.type)) {
|
|
517
|
+
return undefined;
|
|
518
|
+
}
|
|
519
|
+
return renderAddress(tuple);
|
|
520
|
+
};
|
|
521
|
+
|
|
522
|
+
const out = [];
|
|
523
|
+
for (const note of walkMarkdownTree(contentBase, { skipDirectories })) {
|
|
524
|
+
if (!note.frontmatter) continue;
|
|
525
|
+
const file = path.relative(process.cwd(), note.absPath);
|
|
526
|
+
let raw;
|
|
527
|
+
const rawText = () => (raw ??= fs.readFileSync(note.absPath, "utf8"));
|
|
528
|
+
|
|
529
|
+
for (const slot of ART_SLOTS) {
|
|
530
|
+
const value = note.frontmatter.data?.[slot.key];
|
|
531
|
+
if (typeof value !== "string" || !value || value === slot.declines) continue;
|
|
532
|
+
const address = ownAddress(value, slot.type, slot.accepts);
|
|
533
|
+
if (!address) continue;
|
|
534
|
+
out.push({
|
|
535
|
+
file,
|
|
536
|
+
...positionOfFrontmatterPath(rawText(), ["data", slot.key]),
|
|
537
|
+
written: value,
|
|
538
|
+
record: byAddress.get(address) ?? null,
|
|
539
|
+
});
|
|
540
|
+
}
|
|
541
|
+
|
|
542
|
+
const body = note.body ?? "";
|
|
543
|
+
const at = (offset) => bodyPosition(body, offset, note.bodyLine ?? 1, note.bodyColumn ?? 1);
|
|
544
|
+
for (const embed of embedsIn(body)) {
|
|
545
|
+
const address = ownAddress(embed.written, EMBED_DEFAULT_TYPE, undefined);
|
|
546
|
+
if (!address) continue;
|
|
547
|
+
out.push({
|
|
548
|
+
file,
|
|
549
|
+
...at(embed.index),
|
|
550
|
+
written: embed.written,
|
|
551
|
+
record: byAddress.get(address) ?? null,
|
|
552
|
+
});
|
|
553
|
+
}
|
|
554
|
+
for (const image of imagesIn(body)) {
|
|
555
|
+
let forms;
|
|
556
|
+
try {
|
|
557
|
+
forms = resolvePathname(image.src, { contentPackage });
|
|
558
|
+
} catch {
|
|
559
|
+
continue;
|
|
560
|
+
}
|
|
561
|
+
if (forms?.state !== "package" || forms.package !== contentPackage) continue;
|
|
562
|
+
const root = forms.suffix.split("/")[0];
|
|
563
|
+
if (!DEPLOYED_ASSET_TYPES.some((type) => type.root === root)) continue;
|
|
564
|
+
out.push({
|
|
565
|
+
file,
|
|
566
|
+
...at(image.index),
|
|
567
|
+
written: image.src,
|
|
568
|
+
record: byPath.get(forms.suffix) ?? null,
|
|
569
|
+
});
|
|
570
|
+
}
|
|
571
|
+
}
|
|
572
|
+
return out;
|
|
573
|
+
}
|
|
574
|
+
|
|
575
|
+
/**
|
|
576
|
+
* The findings a publish reports against the package's notes.
|
|
577
|
+
*
|
|
578
|
+
* Both are warnings:
|
|
579
|
+
*
|
|
580
|
+
* - a note names a picture the package does not have, so no address answers
|
|
581
|
+
* it and nothing is published for it;
|
|
582
|
+
* - a note names a picture the package leaves out, and the CDN does not hold
|
|
583
|
+
* it, so a world adding the set gets no file for that document.
|
|
584
|
+
*
|
|
585
|
+
* @param {ReturnType<typeof noteArtReferences>} references - From
|
|
586
|
+
* {@link noteArtReferences}.
|
|
587
|
+
* @param {object} opts
|
|
588
|
+
* @param {string} opts.contentPackage - The content package.
|
|
589
|
+
* @param {ReadonlySet<string>} opts.published - Every address on the CDN at
|
|
590
|
+
* its planned size.
|
|
591
|
+
* @param {ReadonlySet<object>} opts.excluded - The records the package leaves
|
|
592
|
+
* out.
|
|
593
|
+
* @returns {Array<{file: string, line?: number, column?: number,
|
|
594
|
+
* severity: "warning", message: string}>} In note order.
|
|
595
|
+
*/
|
|
596
|
+
export function noteArtFindings(references, { contentPackage, published, excluded }) {
|
|
597
|
+
const findings = [];
|
|
598
|
+
for (const { record, written, ...where } of references) {
|
|
599
|
+
if (!record) {
|
|
600
|
+
findings.push({
|
|
601
|
+
...where,
|
|
602
|
+
severity: "warning",
|
|
603
|
+
message:
|
|
604
|
+
`\`${written}\` names a picture the \`${contentPackage}\` package does not ` +
|
|
605
|
+
"have, so no address answers it and the CDN has nothing to publish for it",
|
|
606
|
+
});
|
|
607
|
+
} else if (excluded.has(record) && !published.has(record.address.canonical)) {
|
|
608
|
+
findings.push({
|
|
609
|
+
...where,
|
|
610
|
+
severity: "warning",
|
|
611
|
+
message:
|
|
612
|
+
`\`${record.address.canonical}\` is left out of the package and is not on ` +
|
|
613
|
+
"the CDN, so a world adding this set has no file for it",
|
|
614
|
+
});
|
|
615
|
+
}
|
|
616
|
+
}
|
|
617
|
+
return findings;
|
|
618
|
+
}
|
package/engine/index.mjs
CHANGED
|
@@ -193,6 +193,12 @@ export * as assetTypes from "./asset-types.mjs";
|
|
|
193
193
|
/** The asset record: one index line per addressable file, and its provenance. */
|
|
194
194
|
export * as assetIndex from "./asset-index.mjs";
|
|
195
195
|
|
|
196
|
+
/** The deployed layout of a package's pictures, which of them ship, and its asset set. */
|
|
197
|
+
export * as assetShipping from "./asset-shipping.mjs";
|
|
198
|
+
|
|
199
|
+
/** The address every asset reference in a compiled document states. */
|
|
200
|
+
export * as assetReferences from "./asset-references.mjs";
|
|
201
|
+
|
|
196
202
|
/** The `<package>` segment's registry: the names no repository may claim. */
|
|
197
203
|
export * as packages from "./packages.mjs";
|
|
198
204
|
|