@dogsbay/ssg 0.2.0-beta.135 → 0.2.0-beta.138
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/package/THIRD-PARTY-NOTICES.txt +5 -5
- package/dist/package/browser.mjs +256 -43
- package/dist/package/types/core/build.d.ts +40 -6
- package/dist/package/types/core/config.d.ts +18 -1
- package/dist/package/types/core/dev.d.ts +4 -0
- package/dist/package/types/core/diagrams.d.ts +5 -5
- package/dist/package/types/core/preprocess.d.ts +10 -0
- package/package.json +32 -30
|
@@ -1,31 +1,31 @@
|
|
|
1
1
|
Third-party software in @dogsbay/ssg (dist/package/)
|
|
2
2
|
|
|
3
3
|
========================================================================
|
|
4
|
-
@dogsbay/elements@0.2.0-beta.
|
|
4
|
+
@dogsbay/elements@0.2.0-beta.138 (MIT)
|
|
5
5
|
https://github.com/dogsbay/dogsbay.git
|
|
6
6
|
|
|
7
7
|
Licensed under MIT; the package ships no licence file.
|
|
8
8
|
|
|
9
9
|
========================================================================
|
|
10
|
-
@dogsbay/primitives@0.2.0-beta.
|
|
10
|
+
@dogsbay/primitives@0.2.0-beta.138 (MIT)
|
|
11
11
|
https://github.com/dogsbay/dogsbay.git
|
|
12
12
|
|
|
13
13
|
Licensed under MIT; the package ships no licence file.
|
|
14
14
|
|
|
15
15
|
========================================================================
|
|
16
|
-
@dogsbay/render-html@0.2.0-beta.
|
|
16
|
+
@dogsbay/render-html@0.2.0-beta.138 (MIT)
|
|
17
17
|
https://github.com/dogsbay/dogsbay.git
|
|
18
18
|
|
|
19
19
|
Licensed under MIT; the package ships no licence file.
|
|
20
20
|
|
|
21
21
|
========================================================================
|
|
22
|
-
@dogsbay/search@0.2.0-beta.
|
|
22
|
+
@dogsbay/search@0.2.0-beta.138 (MIT)
|
|
23
23
|
https://github.com/dogsbay/dogsbay.git
|
|
24
24
|
|
|
25
25
|
Licensed under MIT; the package ships no licence file.
|
|
26
26
|
|
|
27
27
|
========================================================================
|
|
28
|
-
@dogsbay/ui@0.2.0-beta.
|
|
28
|
+
@dogsbay/ui@0.2.0-beta.138 (MIT)
|
|
29
29
|
https://github.com/dogsbay/dogsbay.git
|
|
30
30
|
|
|
31
31
|
Licensed under MIT; the package ships no licence file.
|
package/dist/package/browser.mjs
CHANGED
|
@@ -143,7 +143,8 @@ function parseConfig(text, overrides = {}) {
|
|
|
143
143
|
locale: s.locale != null ? unitId("locale", s.locale) : void 0,
|
|
144
144
|
attributes: s.attributes ?? void 0,
|
|
145
145
|
templates: sourceTemplates(s.templates, String(s.path)),
|
|
146
|
-
...s.nav != null ? { nav: sourceNav(s.nav, String(s.path), s.from) } : {}
|
|
146
|
+
...s.nav != null ? { nav: sourceNav(s.nav, String(s.path), s.from) } : {},
|
|
147
|
+
...s.data != null ? { data: dataFolder(`content.sources (${String(s.path)}): data`, s.data) } : {}
|
|
147
148
|
}));
|
|
148
149
|
if (sources.length === 0) sources.push({ path: "content" });
|
|
149
150
|
const versions = (content.versions ?? raw.versions ?? []).map((v) => ({
|
|
@@ -243,6 +244,8 @@ function parseConfig(text, overrides = {}) {
|
|
|
243
244
|
codeBlockTitle: content.codeBlockTitle === false ? false : content.codeBlockTitle === "auto" ? "auto" : true,
|
|
244
245
|
imageCaptions: content.imageCaptions === "title" ? "title" : false,
|
|
245
246
|
html: htmlOf(content.html),
|
|
247
|
+
dataDir: content.data != null ? dataFolder("content.data", content.data) : "data",
|
|
248
|
+
dataDirSet: content.data != null,
|
|
246
249
|
plausible: typeof raw.analytics?.plausible?.domain === "string" ? { domain: raw.analytics.plausible.domain, scriptUrl: strOrUndef(raw.analytics.plausible.scriptUrl) } : void 0,
|
|
247
250
|
...overrides
|
|
248
251
|
};
|
|
@@ -466,7 +469,17 @@ function projectRoots(cfg) {
|
|
|
466
469
|
const roots = cfg.sources.map((s) => s.path === "." ? "" : s.path);
|
|
467
470
|
if (roots.includes("")) return [""];
|
|
468
471
|
const navDirs = cfg.sources.flatMap((s) => s.nav ? [s.nav.includes("/") ? s.nav.slice(0, s.nav.lastIndexOf("/")) : s.nav] : []);
|
|
469
|
-
|
|
472
|
+
const dataDirs = [cfg.dataDir, ...cfg.sources.flatMap((s) => s.data ? [s.data] : [])];
|
|
473
|
+
return [.../* @__PURE__ */ new Set([...roots, cfg.themeDir, ...cfg.css, ...navDirs, ...dataDirs])].filter(Boolean);
|
|
474
|
+
}
|
|
475
|
+
function dataFolder(what, v) {
|
|
476
|
+
if (typeof v !== "string" || !v.trim()) throw new ConfigError(`${what} must be a folder inside the project, such as ./data; got ${JSON.stringify(v)}`);
|
|
477
|
+
const p = insideProject(what, v.trim()).replace(/^(\.\/)+/, "").replace(/\/+$/, "");
|
|
478
|
+
if (!p || p === ".") throw new ConfigError(`${what} must be a folder inside the project, not the project itself (every file would be a data file)`);
|
|
479
|
+
return p;
|
|
480
|
+
}
|
|
481
|
+
function dataDirOf(cfg, s) {
|
|
482
|
+
return s.data ?? cfg.dataDir;
|
|
470
483
|
}
|
|
471
484
|
function sourceRoot(s) {
|
|
472
485
|
return s.path === "." ? "" : s.path.replace(/^\.\//, "").replace(/\/+$/, "") + "/";
|
|
@@ -1072,31 +1085,31 @@ var CLIENT_CANDIDATES = ['""', '"&"', '"\\\\$&")).join("|")})', '"\\\\$&")}"', '
|
|
|
1072
1085
|
var SITE_NOTICES = `Third-party software in this site's _dogsbay/ files (dogsbay.js, site.css) and pages
|
|
1073
1086
|
|
|
1074
1087
|
========================================================================
|
|
1075
|
-
@dogsbay/elements@0.2.0-beta.
|
|
1088
|
+
@dogsbay/elements@0.2.0-beta.138 (MIT)
|
|
1076
1089
|
https://github.com/dogsbay/dogsbay.git
|
|
1077
1090
|
|
|
1078
1091
|
Licensed under MIT; the package ships no licence file.
|
|
1079
1092
|
|
|
1080
1093
|
========================================================================
|
|
1081
|
-
@dogsbay/primitives@0.2.0-beta.
|
|
1094
|
+
@dogsbay/primitives@0.2.0-beta.138 (MIT)
|
|
1082
1095
|
https://github.com/dogsbay/dogsbay.git
|
|
1083
1096
|
|
|
1084
1097
|
Licensed under MIT; the package ships no licence file.
|
|
1085
1098
|
|
|
1086
1099
|
========================================================================
|
|
1087
|
-
@dogsbay/render-html@0.2.0-beta.
|
|
1100
|
+
@dogsbay/render-html@0.2.0-beta.138 (MIT)
|
|
1088
1101
|
https://github.com/dogsbay/dogsbay.git
|
|
1089
1102
|
|
|
1090
1103
|
Licensed under MIT; the package ships no licence file.
|
|
1091
1104
|
|
|
1092
1105
|
========================================================================
|
|
1093
|
-
@dogsbay/search@0.2.0-beta.
|
|
1106
|
+
@dogsbay/search@0.2.0-beta.138 (MIT)
|
|
1094
1107
|
https://github.com/dogsbay/dogsbay.git
|
|
1095
1108
|
|
|
1096
1109
|
Licensed under MIT; the package ships no licence file.
|
|
1097
1110
|
|
|
1098
1111
|
========================================================================
|
|
1099
|
-
@dogsbay/ui@0.2.0-beta.
|
|
1112
|
+
@dogsbay/ui@0.2.0-beta.138 (MIT)
|
|
1100
1113
|
https://github.com/dogsbay/dogsbay.git
|
|
1101
1114
|
|
|
1102
1115
|
Licensed under MIT; the package ships no licence file.
|
|
@@ -2253,6 +2266,7 @@ async function preprocess(text, path, attributes, files, undefinedRefs = "preser
|
|
|
2253
2266
|
};
|
|
2254
2267
|
if (budget && timeout <= 0) return skip();
|
|
2255
2268
|
budget?.start();
|
|
2269
|
+
const warnings = [];
|
|
2256
2270
|
try {
|
|
2257
2271
|
const out = await render(mask(text), {
|
|
2258
2272
|
context: withUnderscoreAliases(attributes),
|
|
@@ -2262,9 +2276,14 @@ async function preprocess(text, path, attributes, files, undefinedRefs = "preser
|
|
|
2262
2276
|
basePath: dirname(path),
|
|
2263
2277
|
loader: new MapLoader(files),
|
|
2264
2278
|
timeout,
|
|
2265
|
-
...run.parseCache !== void 0 ? { parseCache: run.parseCache } : {}
|
|
2279
|
+
...run.parseCache !== void 0 ? { parseCache: run.parseCache } : {},
|
|
2280
|
+
// Once per tag and place: a page that includes a file twice is told once.
|
|
2281
|
+
onWarning: (w) => {
|
|
2282
|
+
if (!warnings.some((x) => x.tag === w.tag && x.line === w.line && x.include === w.include && x.variable === w.variable))
|
|
2283
|
+
warnings.push({ tag: w.tag, line: w.line, ...w.include ? { include: w.include } : {}, ...w.variable ? { variable: w.variable } : {} });
|
|
2284
|
+
}
|
|
2266
2285
|
});
|
|
2267
|
-
return { text: stripFenceMinjaMarkers(stripInlineMinjaMarkers(unmask(out))) };
|
|
2286
|
+
return { text: stripFenceMinjaMarkers(stripInlineMinjaMarkers(unmask(out))), ...warnings.length ? { warnings } : {} };
|
|
2268
2287
|
} catch (err) {
|
|
2269
2288
|
if (budget && timeout < budget.renderMs && err instanceof RenderLimitError && budget.next() <= 0) return skip();
|
|
2270
2289
|
return { text, error: err instanceof Error ? err.message : String(err) };
|
|
@@ -2360,7 +2379,7 @@ var ScanCache = class {
|
|
|
2360
2379
|
};
|
|
2361
2380
|
|
|
2362
2381
|
// src/core/build.ts
|
|
2363
|
-
import { TemplatePages } from "@dogsbay/format-dogsbay-md/browser";
|
|
2382
|
+
import { DATA_DIRECTIVES, DataCache, DataFileError, DataFiles, DEFAULT_DATA_LIMITS, hasDataDirectives, NO_PROJECT_MESSAGE, resolveDataDirectives, TemplatePages, UNTRUSTED_DATA_LIMITS } from "@dogsbay/format-dogsbay-md/browser";
|
|
2364
2383
|
import { extractText, SearchIndexBuilder } from "@dogsbay/search/index";
|
|
2365
2384
|
|
|
2366
2385
|
// src/core/strings.ts
|
|
@@ -2981,7 +3000,7 @@ function reporter(warnings, problems) {
|
|
|
2981
3000
|
problems.push({ code, ...at, ...at.file !== void 0 ? { file: detach(at.file) } : {}, message, text: t });
|
|
2982
3001
|
};
|
|
2983
3002
|
}
|
|
2984
|
-
function linedDiagnostics(found, raw, body, parsed) {
|
|
3003
|
+
function linedDiagnostics(found, raw, body, parsed, after = "of the page after templating") {
|
|
2985
3004
|
if (!found.length) return [];
|
|
2986
3005
|
const offset = raw.slice(0, raw.length - body.length).split("\n").length - 1;
|
|
2987
3006
|
const LINE = /\bline (\d+)/g;
|
|
@@ -2999,10 +3018,34 @@ function linedDiagnostics(found, raw, body, parsed) {
|
|
|
2999
3018
|
return found.map((d) => {
|
|
3000
3019
|
const i = asWritten.findIndex((w) => key(w) === key(d));
|
|
3001
3020
|
if (i >= 0) return lined(asWritten.splice(i, 1)[0]);
|
|
3002
|
-
return { code: d.code, message: detach(d.message.replace(LINE,
|
|
3021
|
+
return { code: d.code, message: detach(d.message.replace(LINE, `line $1 ${after}`)), ...named(d) };
|
|
3003
3022
|
});
|
|
3004
3023
|
}
|
|
3005
3024
|
var decoder = new TextDecoder();
|
|
3025
|
+
var DATA_DIRECTIVE_NODES = new Set(DATA_DIRECTIVES);
|
|
3026
|
+
var DEFAULT_DATA_CACHE_BYTES = 32 * 1024 * 1024;
|
|
3027
|
+
var NO_DATA_FOLDER = "this site has no data folder (data/ beside dogsbay.config.yml, or content.data)";
|
|
3028
|
+
function hasFilesUnder(files, dir) {
|
|
3029
|
+
const prefix = `${dir}/`;
|
|
3030
|
+
for (const k of files.keys()) if (k.startsWith(prefix)) return true;
|
|
3031
|
+
return false;
|
|
3032
|
+
}
|
|
3033
|
+
function strictestData(floor, given) {
|
|
3034
|
+
const out = { ...floor };
|
|
3035
|
+
for (const k of Object.keys(floor)) {
|
|
3036
|
+
const g = given?.[k];
|
|
3037
|
+
if (typeof g === "number" && Number.isFinite(g) && g >= 0 && g < floor[k]) out[k] = g;
|
|
3038
|
+
}
|
|
3039
|
+
return out;
|
|
3040
|
+
}
|
|
3041
|
+
function finiteLimits(given) {
|
|
3042
|
+
const out = {};
|
|
3043
|
+
for (const [k, v] of Object.entries(given ?? {})) if (typeof v === "number" && Number.isFinite(v) && v >= 0) out[k] = v;
|
|
3044
|
+
return out;
|
|
3045
|
+
}
|
|
3046
|
+
function inDataFolder(cfg, path) {
|
|
3047
|
+
return [cfg.dataDir, ...cfg.sources.flatMap((s) => s.data ? [s.data] : [])].some((d) => path.startsWith(`${d}/`));
|
|
3048
|
+
}
|
|
3006
3049
|
async function readText(c) {
|
|
3007
3050
|
if (typeof c === "string") return c;
|
|
3008
3051
|
if (isLazyText(c)) return c.read();
|
|
@@ -3082,13 +3125,16 @@ async function build(input, hooks = {}) {
|
|
|
3082
3125
|
const problems = [];
|
|
3083
3126
|
const report = reporter(warnings, problems);
|
|
3084
3127
|
for (const m of ignored) report("untrusted", `[untrusted] ${m}`);
|
|
3128
|
+
for (const [what, dir] of [...cfg.dataDirSet ? [["content.data", cfg.dataDir]] : [], ...cfg.sources.flatMap((s) => s.data ? [[`content.sources (${s.path}) data`, s.data]] : [])]) {
|
|
3129
|
+
if (!hasFilesUnder(input.files, dir)) report("data", `[data] ${what}: ${dir} has no files, so its data directives and data.<name> lookups find nothing`);
|
|
3130
|
+
}
|
|
3085
3131
|
for (const p of cfg.unsupportedPlugins) report("plugins", `[plugins] ${p} is an Astro-pipeline plugin; dogsbay-ssg ignores it`);
|
|
3086
3132
|
const highlighter = input.highlighter ?? await (await import("@dogsbay/highlight")).createHighlighter({ cacheChars: input.highlightCacheChars });
|
|
3087
3133
|
const buildCss = await siteCssBuilder(input);
|
|
3088
3134
|
phase.init = now() - t0;
|
|
3089
3135
|
let pages = 0;
|
|
3090
3136
|
let assets = 0;
|
|
3091
|
-
const state = { redirects: [], navCache: navCache(), written: /* @__PURE__ */ new Set(), stubs: [], pageOut: /* @__PURE__ */ new Map(), sitemaps: /* @__PURE__ */ new Map(), supplied: /* @__PURE__ */ new Set(), usedMath: false, diagrams: diagramStore(input, cfg), scan: hooks.keep || hooks.lazy ? new ScanCache() : void 0, parseCache: input.parseCacheBytes === 0 ? false : new ParseCache(input.parseCacheBytes ?? DEFAULT_PARSE_CACHE_BYTES), templateBudget: templateBudget(input.untrusted === true, input.templateMs) };
|
|
3137
|
+
const state = { redirects: [], navCache: navCache(), written: /* @__PURE__ */ new Set(), stubs: [], pageOut: /* @__PURE__ */ new Map(), sitemaps: /* @__PURE__ */ new Map(), supplied: /* @__PURE__ */ new Set(), usedMath: false, diagrams: diagramStore(input, cfg), scan: hooks.keep || hooks.lazy ? new ScanCache() : void 0, parseCache: input.parseCacheBytes === 0 ? false : new ParseCache(input.parseCacheBytes ?? DEFAULT_PARSE_CACHE_BYTES), templateBudget: templateBudget(input.untrusted === true, input.templateMs), dataCache: new DataCache(input.dataCacheBytes ?? DEFAULT_DATA_CACHE_BYTES) };
|
|
3092
3138
|
for (const r of cfg.redirects) {
|
|
3093
3139
|
const from = sitePath(r.from, cfg.basePath);
|
|
3094
3140
|
if (!from) {
|
|
@@ -3279,8 +3325,8 @@ async function buildUnit(unit, allUnits, cfg, input, highlighter, emit, report,
|
|
|
3279
3325
|
const prefix = unit.prefix ? unit.prefix + "/" : "";
|
|
3280
3326
|
const label = unit.prefix || "/";
|
|
3281
3327
|
const warn = (code, m, at = {}) => report(code, `[${label}] ${m}`, { unit: label, ...at, message: at.message ?? m });
|
|
3282
|
-
const warnFile = (code, shown, file, m, line) => warn(code, `${shown}: ${m}`, { file, message: m, ...line ? { line } : {} });
|
|
3283
|
-
const warnPage = (info, code, m, line) => warnFile(code, where(info), srcOf(info), m, line);
|
|
3328
|
+
const warnFile = (code, shown, file, m, line, more = {}) => warn(code, `${shown}: ${m}`, { file, message: m, ...line ? { line } : {}, ...more });
|
|
3329
|
+
const warnPage = (info, code, m, line, more) => warnFile(code, where(info), srcOf(info), m, line, more);
|
|
3284
3330
|
const navFileAt = () => navProjectFiles.length === 1 ? { file: navProjectFiles[0] } : {};
|
|
3285
3331
|
const diagramWarn = (m, file) => file ? warnFile("diagram", file, abs(file), m) : warn("diagram", m);
|
|
3286
3332
|
let math;
|
|
@@ -3385,13 +3431,54 @@ ${marker}${title}${marker}`, path, attributesOf(path), includesFor(path, read2),
|
|
|
3385
3431
|
if (k === void 0) resolvedKeys.set(key, k = JSON.stringify([label, key, sourceOf(p).templates ?? null, cfg.name, cfg.undefinedRefs, attributesOf(p)]));
|
|
3386
3432
|
return k;
|
|
3387
3433
|
};
|
|
3388
|
-
const
|
|
3434
|
+
const dataLimits = input.untrusted ? strictestData(UNTRUSTED_DATA_LIMITS, input.dataLimits) : { ...DEFAULT_DATA_LIMITS, ...finiteLimits(input.dataLimits) };
|
|
3435
|
+
const pageDataBytes = Math.min(dataLimits.maxTotalBytes, 4 * dataLimits.maxFileBytes);
|
|
3436
|
+
let dataDirHas = /* @__PURE__ */ new Map();
|
|
3437
|
+
const dataDir = (p) => {
|
|
3438
|
+
const dir2 = dataDirOf(cfg, sourceOf(p));
|
|
3439
|
+
let has = dataDirHas.get(dir2);
|
|
3440
|
+
if (has === void 0) dataDirHas.set(dir2, has = hasFilesUnder(input.files, dir2));
|
|
3441
|
+
return has ? dir2 : void 0;
|
|
3442
|
+
};
|
|
3443
|
+
const dataReader = (dir2, onLoad) => new DataFiles({
|
|
3444
|
+
limits: dataLimits,
|
|
3445
|
+
cache: state.dataCache,
|
|
3446
|
+
load: async (rel2) => {
|
|
3447
|
+
const c = input.files.get(`${dir2}/${rel2}`);
|
|
3448
|
+
if (c == null) return void 0;
|
|
3449
|
+
const text = await readText(c);
|
|
3450
|
+
onLoad?.(text.length, rel2);
|
|
3451
|
+
return text;
|
|
3452
|
+
}
|
|
3453
|
+
});
|
|
3454
|
+
let dataBy = /* @__PURE__ */ new Map();
|
|
3455
|
+
const dataOf = (p) => {
|
|
3456
|
+
const dir2 = dataDir(p);
|
|
3457
|
+
if (!dir2) return void 0;
|
|
3458
|
+
let d = dataBy.get(dir2);
|
|
3459
|
+
if (!d) dataBy.set(dir2, d = dataReader(dir2));
|
|
3460
|
+
return d;
|
|
3461
|
+
};
|
|
3462
|
+
const pageDataOf = (p) => {
|
|
3463
|
+
const dir2 = dataDir(p);
|
|
3464
|
+
if (!dir2) return void 0;
|
|
3465
|
+
let read2 = 0;
|
|
3466
|
+
return dataReader(dir2, (bytes, rel2) => {
|
|
3467
|
+
read2 += bytes;
|
|
3468
|
+
if (read2 > pageDataBytes) throw new DataFileError(`this page's data directives read more than ${pageDataBytes.toLocaleString("en")} bytes of data in all (each directive counts its file again); split the page or the file`, rel2);
|
|
3469
|
+
});
|
|
3470
|
+
};
|
|
3471
|
+
const dataUnresolved = /* @__PURE__ */ new Set();
|
|
3472
|
+
const dataReads = /* @__PURE__ */ new Map();
|
|
3473
|
+
const dataTemplatePages = /* @__PURE__ */ new Set();
|
|
3474
|
+
let dataInTitles = false;
|
|
3475
|
+
let templatesBy = /* @__PURE__ */ new Map();
|
|
3389
3476
|
const templatesOf = (p) => {
|
|
3390
3477
|
const key = named.length ? namespaceOf(p) : "";
|
|
3391
3478
|
let tp = templatesBy.get(key);
|
|
3392
3479
|
if (!tp) {
|
|
3393
3480
|
const at = scoped(p);
|
|
3394
|
-
tp = new TemplatePages({ load: (q) => readInclude(at ? at(q) : q), attributes: attributesOf(p), sourceDefault: sourceOf(p).templates === "jinja" });
|
|
3481
|
+
tp = new TemplatePages({ load: (q) => readInclude(at ? at(q) : q), attributes: attributesOf(p), sourceDefault: sourceOf(p).templates === "jinja", data: dataOf(p) });
|
|
3395
3482
|
templatesBy.set(key, tp);
|
|
3396
3483
|
}
|
|
3397
3484
|
return tp;
|
|
@@ -3409,6 +3496,7 @@ ${marker}${title}${marker}`, path, attributesOf(path), includesFor(path, read2),
|
|
|
3409
3496
|
const leftOut = /* @__PURE__ */ new Map();
|
|
3410
3497
|
for (const [p, c] of files) {
|
|
3411
3498
|
if (buildOnly(p)) continue;
|
|
3499
|
+
if (inDataFolder(cfg, abs(p))) continue;
|
|
3412
3500
|
if (/\.(md|markdown)$/i.test(p)) {
|
|
3413
3501
|
if (excluded(p)) {
|
|
3414
3502
|
leftOut.set(slugFromPath(p), cfg.excludeFromRoutes.find((d) => p === d || p.startsWith(d.replace(/\/+$/, "") + "/")) ?? "_ or . files");
|
|
@@ -3473,6 +3561,7 @@ ${marker}${title}${marker}`, path, attributesOf(path), includesFor(path, read2),
|
|
|
3473
3561
|
for (const f of memo2.deps ?? []) includedProject.add(f);
|
|
3474
3562
|
} else {
|
|
3475
3563
|
const keep = memo2 && !templatesOf(p).isTemplatePage(fm);
|
|
3564
|
+
if (templatesOf(p).isTemplatePage(fm) && /\bdata\s*[.[]/.test(`${str(fm.title) ?? ""} ${fm.description ?? ""}`)) dataInTitles = true;
|
|
3476
3565
|
const reads = keep ? (f) => state.scan.read(abs(p), f) : void 0;
|
|
3477
3566
|
const skippedBefore = state.templateBudget.skipped.titles;
|
|
3478
3567
|
title = detach(await resolveTitle(str(fm.title) ?? firstH1(await source()) ?? humanize2(slug.split("/").pop() || cfg.name), await source(), p, fm, reads));
|
|
@@ -3708,7 +3797,7 @@ ${marker}${title}${marker}`, path, attributesOf(path), includesFor(path, read2),
|
|
|
3708
3797
|
const brokenCounts = /* @__PURE__ */ new Map();
|
|
3709
3798
|
const prepare = async (info) => {
|
|
3710
3799
|
const warned = [];
|
|
3711
|
-
const warn2 = (code, m, line) => warned.push([code, m, line]);
|
|
3800
|
+
const warn2 = (code, m, line, more) => warned.push([code, m, line, more]);
|
|
3712
3801
|
let diagnostics;
|
|
3713
3802
|
let t2 = now();
|
|
3714
3803
|
if (info.virtual) {
|
|
@@ -3730,18 +3819,54 @@ ${marker}${title}${marker}`, path, attributesOf(path), includesFor(path, read2),
|
|
|
3730
3819
|
warn2("front-matter", `front matter does not parse${line ? ` (line ${line})` : ""}: ${reason}`, line);
|
|
3731
3820
|
if (mode.diagnostics) diagnostics = [{ code: "front-matter", message: detach(`front matter does not parse: ${reason}`), ...line ? { line } : {} }];
|
|
3732
3821
|
}
|
|
3733
|
-
const
|
|
3822
|
+
const reads = /* @__PURE__ */ new Set();
|
|
3823
|
+
const readDir = dataDir(info.source);
|
|
3824
|
+
const isTemplate = templatesOf(info.source).isTemplatePage(fm);
|
|
3825
|
+
if (isTemplate && readDir) dataTemplatePages.add(info.source);
|
|
3826
|
+
else dataTemplatePages.delete(info.source);
|
|
3827
|
+
const pre = isTemplate ? { text: await templatesOf(info.source).render(bodySrc, info.source, fm, { firstLine: raw.slice(0, raw.length - bodySrc.length).split("\n").length, onDataRead: (d) => reads.add(`${readDir}/${d}`) }) } : await preprocess(bodySrc, info.source, attributesOf(info.source), includesFor(info.source), cfg.undefinedRefs, templateRun);
|
|
3734
3828
|
if (pre.error) {
|
|
3735
3829
|
warn2("template", `template error: ${pre.error}`);
|
|
3736
3830
|
if (mode.diagnostics) (diagnostics ??= []).push({ code: "template", message: detach(`template error: ${pre.error}`) });
|
|
3737
3831
|
}
|
|
3832
|
+
if ("warnings" in pre && pre.warnings) {
|
|
3833
|
+
const fmLines = raw.slice(0, raw.length - bodySrc.length).split("\n").length - 1;
|
|
3834
|
+
for (const w of pre.warnings) {
|
|
3835
|
+
if (w.tag.startsWith("end")) continue;
|
|
3836
|
+
const where2 = w.include ? ` (in ${w.include})` : w.variable ? ` (in the value of {{ ${w.variable} }})` : "";
|
|
3837
|
+
const m = `{% ${w.tag} %} is not a tag of an ordinary page and is shown as written${where2}; for loops and the like, make it a template page (template: jinja in its front matter)`;
|
|
3838
|
+
const line = w.include || w.variable ? void 0 : w.line + fmLines;
|
|
3839
|
+
warn2("template-tag", m, line);
|
|
3840
|
+
if (mode.diagnostics) (diagnostics ??= []).push({ code: "template-tag", message: detach(m), ...line ? { line } : {} });
|
|
3841
|
+
}
|
|
3842
|
+
}
|
|
3843
|
+
const templated = pre.text;
|
|
3844
|
+
dataUnresolved.delete(info.source);
|
|
3845
|
+
if (hasDataDirectives(pre.text)) {
|
|
3846
|
+
const data = pageDataOf(info.source);
|
|
3847
|
+
const r = await resolveDataDirectives(pre.text, data);
|
|
3848
|
+
if (r.problems.length) dataUnresolved.add(info.source);
|
|
3849
|
+
for (const d of r.reads) reads.add(`${readDir}/${d}`);
|
|
3850
|
+
const offset = pre.text === bodySrc ? raw.slice(0, raw.length - bodySrc.length).split("\n").length - 1 : void 0;
|
|
3851
|
+
for (const pr of r.problems) {
|
|
3852
|
+
const message = data ? pr.message : pr.message.replace(NO_PROJECT_MESSAGE, NO_DATA_FOLDER);
|
|
3853
|
+
const line = offset === void 0 ? void 0 : pr.line + offset;
|
|
3854
|
+
const shown = line === void 0 ? `${message} (line ${pr.line} of the page after templating)` : message;
|
|
3855
|
+
const inFile = pr.file && readDir && data && !/(^|\/)\.\.?(\/|$)/.test(pr.file) && input.files.has(`${readDir}/${pr.file}`) ? { dataFile: `${readDir}/${pr.file}`, ...pr.fileLine ? { dataLine: pr.fileLine } : {} } : {};
|
|
3856
|
+
warn2("data", shown, line, inFile);
|
|
3857
|
+
if (mode.diagnostics) (diagnostics ??= []).push({ code: "data", message: detach(shown), ...line ? { line } : {} });
|
|
3858
|
+
}
|
|
3859
|
+
pre.text = r.text;
|
|
3860
|
+
}
|
|
3861
|
+
if (reads.size) dataReads.set(info.source, reads);
|
|
3862
|
+
else dataReads.delete(info.source);
|
|
3738
3863
|
phase.template = (phase.template ?? 0) + now() - t2;
|
|
3739
3864
|
t2 = now();
|
|
3740
3865
|
let tree;
|
|
3741
3866
|
try {
|
|
3742
3867
|
const parsed = dogsbayMdToTree(pre.text, mode.diagnostics ? { diagnostics: true } : void 0);
|
|
3743
3868
|
tree = parsed.tree;
|
|
3744
|
-
if (mode.diagnostics) (diagnostics ??= []).push(...linedDiagnostics(parsed.diagnostics ?? [], raw, bodySrc, pre.text));
|
|
3869
|
+
if (mode.diagnostics) (diagnostics ??= []).push(...linedDiagnostics(parsed.diagnostics ?? [], raw, bodySrc, pre.text, templated === bodySrc ? "of the page with its data directives drawn" : void 0));
|
|
3745
3870
|
} catch (e) {
|
|
3746
3871
|
warn2("parse", `parse error: ${e.message}`);
|
|
3747
3872
|
if (mode.diagnostics) (diagnostics ??= []).push({ code: "parse", message: detach(`parse error: ${e.message}`) });
|
|
@@ -3759,50 +3884,100 @@ ${marker}${title}${marker}`, path, attributesOf(path), includesFor(path, read2),
|
|
|
3759
3884
|
const markdown = (unresolved || ns) && cfg.mdMirror ? treeToDogsbayMd(tree) : pre.text;
|
|
3760
3885
|
return { raw, fm, bodySrc, pre, tree, warned, markdown, diagnostics };
|
|
3761
3886
|
};
|
|
3762
|
-
const
|
|
3887
|
+
const pagesOpt = Math.floor(input.highlightAhead?.pages ?? 24);
|
|
3888
|
+
const aheadPages = Number.isNaN(pagesOpt) ? 24 : Math.max(1, pagesOpt);
|
|
3889
|
+
const aheadChars = input.highlightAhead?.chars ?? Infinity;
|
|
3890
|
+
const charBound = Number.isFinite(aheadChars);
|
|
3891
|
+
const keyOf = (code, lang2, opts) => `${lang2 ?? ""}\0${JSON.stringify(opts)}\0${code}`;
|
|
3892
|
+
const delivered = /* @__PURE__ */ new Map();
|
|
3893
|
+
const wanted = /* @__PURE__ */ new Map();
|
|
3894
|
+
const pageHighlighter = input.highlightPool ? {
|
|
3895
|
+
render: (c, l, o) => highlighter.render(c, l, o),
|
|
3896
|
+
code: (c, l, o) => delivered.get(keyOf(c, l, o)) ?? highlighter.code(c, l, o),
|
|
3897
|
+
resolve: (l) => highlighter.resolve(l),
|
|
3898
|
+
has: (c, l, o) => delivered.has(keyOf(c, l, o)) || highlighter.has(c, l, o),
|
|
3899
|
+
prime: (c, l, o, h) => highlighter.prime(c, l, o, h),
|
|
3900
|
+
cacheStats: () => highlighter.cacheStats(),
|
|
3901
|
+
load: highlighter.load && ((b) => highlighter.load(b))
|
|
3902
|
+
} : highlighter;
|
|
3763
3903
|
const renderAhead = async (all, pool) => {
|
|
3764
3904
|
const list = input.onPageBuilt ? all : all.filter((info) => !(replaced != null && info.slug === replaced));
|
|
3765
3905
|
const inflight = /* @__PURE__ */ new Map();
|
|
3766
3906
|
const ahead = /* @__PURE__ */ new Map();
|
|
3767
|
-
|
|
3907
|
+
let cur = 0;
|
|
3908
|
+
let next = 0;
|
|
3909
|
+
let held = 0;
|
|
3910
|
+
let preparing = 0;
|
|
3911
|
+
const fill = () => {
|
|
3912
|
+
while (next < list.length && next < cur + aheadPages && (next <= cur + 1 || !charBound || preparing === 0 && held < aheadChars)) start(next++);
|
|
3913
|
+
};
|
|
3768
3914
|
const start = (i) => {
|
|
3769
|
-
|
|
3770
|
-
const
|
|
3915
|
+
preparing++;
|
|
3916
|
+
const prepared = prepare(list[i]).finally(() => preparing--);
|
|
3917
|
+
const job = prepared.then(async (p) => {
|
|
3918
|
+
held += p.raw.length;
|
|
3771
3919
|
const waits = [];
|
|
3772
3920
|
const send = [];
|
|
3921
|
+
const keys = /* @__PURE__ */ new Map();
|
|
3773
3922
|
const sendKeys = /* @__PURE__ */ new Set();
|
|
3774
3923
|
for (const r of codeRequests(p.tree, !!diagrams)) {
|
|
3775
|
-
const k = keyOf(r);
|
|
3776
|
-
|
|
3777
|
-
if (
|
|
3778
|
-
|
|
3924
|
+
const k = keyOf(r.code, r.lang, r.opts);
|
|
3925
|
+
if (keys.has(k)) continue;
|
|
3926
|
+
if (delivered.has(k) || inflight.has(k)) {
|
|
3927
|
+
keys.set(k, r);
|
|
3928
|
+
const on = inflight.get(k);
|
|
3929
|
+
if (on) waits.push(on);
|
|
3930
|
+
} else if (!highlighter.has(r.code, r.lang, r.opts)) {
|
|
3931
|
+
keys.set(k, r);
|
|
3779
3932
|
send.push(r);
|
|
3780
3933
|
sendKeys.add(k);
|
|
3781
3934
|
}
|
|
3782
3935
|
}
|
|
3936
|
+
for (const [k, r] of keys) {
|
|
3937
|
+
const w = wanted.get(k);
|
|
3938
|
+
if (w) w.n++;
|
|
3939
|
+
else wanted.set(k, { r, n: 1 });
|
|
3940
|
+
}
|
|
3783
3941
|
if (send.length) {
|
|
3784
|
-
const
|
|
3785
|
-
(html) => send.forEach((r, n) =>
|
|
3942
|
+
const done2 = pool.render(send).then(
|
|
3943
|
+
(html) => send.forEach((r, n) => {
|
|
3944
|
+
const k = keyOf(r.code, r.lang, r.opts);
|
|
3945
|
+
if (typeof html[n] === "string" && wanted.has(k) && !delivered.has(k)) {
|
|
3946
|
+
delivered.set(k, html[n]);
|
|
3947
|
+
held += html[n].length;
|
|
3948
|
+
}
|
|
3949
|
+
}),
|
|
3786
3950
|
() => {
|
|
3787
3951
|
}
|
|
3788
|
-
);
|
|
3789
|
-
const done2 = delivered.finally(() => sendKeys.forEach((k) => inflight.delete(k)));
|
|
3952
|
+
).finally(() => sendKeys.forEach((k) => inflight.delete(k)));
|
|
3790
3953
|
for (const k of sendKeys) inflight.set(k, done2);
|
|
3791
3954
|
waits.push(done2);
|
|
3792
3955
|
}
|
|
3956
|
+
fill();
|
|
3793
3957
|
await Promise.all(waits);
|
|
3794
|
-
return p;
|
|
3958
|
+
return { p, keys };
|
|
3795
3959
|
});
|
|
3796
3960
|
job.catch(() => {
|
|
3797
3961
|
});
|
|
3798
3962
|
ahead.set(i, job);
|
|
3799
3963
|
};
|
|
3800
3964
|
try {
|
|
3801
|
-
for (
|
|
3802
|
-
|
|
3803
|
-
const p = await ahead.get(
|
|
3804
|
-
ahead.delete(
|
|
3805
|
-
await renderPage(list[
|
|
3965
|
+
for (cur = 0; cur < list.length; cur++) {
|
|
3966
|
+
fill();
|
|
3967
|
+
const { p, keys } = await ahead.get(cur);
|
|
3968
|
+
ahead.delete(cur);
|
|
3969
|
+
await renderPage(list[cur], p);
|
|
3970
|
+
held -= p.raw.length;
|
|
3971
|
+
for (const k of keys.keys()) {
|
|
3972
|
+
const w = wanted.get(k);
|
|
3973
|
+
if (--w.n) continue;
|
|
3974
|
+
wanted.delete(k);
|
|
3975
|
+
const html = delivered.get(k);
|
|
3976
|
+
if (html === void 0) continue;
|
|
3977
|
+
delivered.delete(k);
|
|
3978
|
+
held -= html.length;
|
|
3979
|
+
highlighter.prime(w.r.code, w.r.lang, w.r.opts, html);
|
|
3980
|
+
}
|
|
3806
3981
|
}
|
|
3807
3982
|
} finally {
|
|
3808
3983
|
for (const p of ahead.values()) p.catch(() => {
|
|
@@ -3813,7 +3988,7 @@ ${marker}${title}${marker}`, path, attributesOf(path), includesFor(path, read2),
|
|
|
3813
3988
|
const isReplaced = replaced != null && info.slug === replaced;
|
|
3814
3989
|
if (isReplaced && !input.onPageBuilt) return;
|
|
3815
3990
|
const got = prepared ?? await prepare(info);
|
|
3816
|
-
for (const [code, m, line] of got.warned) warnPage(info, code, m, line);
|
|
3991
|
+
for (const [code, m, line, more] of got.warned) warnPage(info, code, m, line, more);
|
|
3817
3992
|
mode.diagnostics?.(srcOf(info), got.diagnostics ?? []);
|
|
3818
3993
|
const { raw, fm, bodySrc, pre, markdown } = got;
|
|
3819
3994
|
let tree = got.tree;
|
|
@@ -3852,7 +4027,7 @@ ${marker}${title}${marker}`, path, attributesOf(path), includesFor(path, read2),
|
|
|
3852
4027
|
const noteRemoved = (code, m) => void (code !== "image-remote" && removed.add(m));
|
|
3853
4028
|
if (sanitizing) tree = sanitizeTree(tree, noteRemoved);
|
|
3854
4029
|
if (cfg.math && !math && mayHaveMath(tree)) math = await loadMath();
|
|
3855
|
-
await highlighter.load?.(highlightInputs(tree));
|
|
4030
|
+
if (!input.highlightPool || codeRequests(tree, !!diagrams).some((r) => !pageHighlighter.has(r.code, r.lang, r.opts))) await highlighter.load?.(highlightInputs(tree));
|
|
3856
4031
|
const unloadedBefore = highlighter.cacheStats().unloaded ?? 0;
|
|
3857
4032
|
t = now();
|
|
3858
4033
|
const apiLayout = apiPageLayout(tree);
|
|
@@ -3860,7 +4035,7 @@ ${marker}${title}${marker}`, path, attributesOf(path), includesFor(path, read2),
|
|
|
3860
4035
|
const ctx = createRenderContext({
|
|
3861
4036
|
link: resolver.link,
|
|
3862
4037
|
asset: resolver.asset,
|
|
3863
|
-
highlighter,
|
|
4038
|
+
highlighter: pageHighlighter,
|
|
3864
4039
|
// The layout's own ids, outside the rendered content: generated ids avoid them
|
|
3865
4040
|
reservedIds: ["db-content", "db-sidebar", "db-search", "db-search-results", "db-toc-title", "db-page-markdown"],
|
|
3866
4041
|
t: t_,
|
|
@@ -3894,7 +4069,7 @@ ${marker}${title}${marker}`, path, attributesOf(path), includesFor(path, read2),
|
|
|
3894
4069
|
if (navNode?.children.length && !hasProse(tree)) body += childLinks2(navNode.children, (s) => pageRefs.get(s), info.out, cfg.linkStyle, infos);
|
|
3895
4070
|
const layoutName = str(fm.layout) ?? (post ? "blank" : cfg.layout);
|
|
3896
4071
|
const tocMode = fm.toc === false ? "off" : ["top", "popover", "rail", "off"].includes(fm.toc) ? fm.toc : cfg.toc;
|
|
3897
|
-
for (const u of ctx.unknown) warnPage(info, "unsupported-node", `unsupported node "${u}" rendered as a plain container`);
|
|
4072
|
+
for (const u of ctx.unknown) if (!(DATA_DIRECTIVE_NODES.has(u) && dataUnresolved.has(info.source))) warnPage(info, "unsupported-node", `unsupported node "${u}" rendered as a plain container`);
|
|
3898
4073
|
for (const u of ctx.unsafeUrls ?? []) warnPage(info, "unsafe-url", `refused the unsafe URL "${u}"`);
|
|
3899
4074
|
for (const m of removed) warnPage(info, "html", m);
|
|
3900
4075
|
for (const name of ctx.missingIcons ?? []) warnPage(info, "icon", `unknown icon "${name}"${input.icons ? "" : " (only the built-in icons are available in this build)"}`);
|
|
@@ -4081,7 +4256,8 @@ ${marker}${title}${marker}`, path, attributesOf(path), includesFor(path, read2),
|
|
|
4081
4256
|
body = sp.content;
|
|
4082
4257
|
} catch {
|
|
4083
4258
|
}
|
|
4084
|
-
const
|
|
4259
|
+
const templated = templatesOf(info.source).isTemplatePage(pageFm) ? await templatesOf(info.source).render(body, info.source, pageFm) : (await preprocess(body, info.source, attributesOf(info.source), includesFor(info.source), cfg.undefinedRefs, { ...templateRun, kind: "copy" })).text;
|
|
4260
|
+
const text = hasDataDirectives(templated) ? (await resolveDataDirectives(templated, pageDataOf(info.source))).text : templated;
|
|
4085
4261
|
const ns = named.length ? namespaceOf(info.source) : "";
|
|
4086
4262
|
if (!ns && !PLACEHOLDER_DIRECTIVE.test(text)) return text;
|
|
4087
4263
|
const tree = dogsbayMdToTree(text).tree;
|
|
@@ -4145,6 +4321,26 @@ ${marker}${title}${marker}`, path, attributesOf(path), includesFor(path, read2),
|
|
|
4145
4321
|
included(path) {
|
|
4146
4322
|
const r = rel(path);
|
|
4147
4323
|
return r !== void 0 && includedPaths.has(r) || includedProject.has(path);
|
|
4324
|
+
},
|
|
4325
|
+
dataReaders(path) {
|
|
4326
|
+
const stem = (p) => p.replace(/\.[A-Za-z]+$/, "");
|
|
4327
|
+
const out = [];
|
|
4328
|
+
for (const info of bySource.values()) {
|
|
4329
|
+
if (info.slug === replaced) continue;
|
|
4330
|
+
const dir2 = dataDirOf(cfg, sourceOf(info.source));
|
|
4331
|
+
if (!path.startsWith(`${dir2}/`)) continue;
|
|
4332
|
+
const read2 = [...dataReads.get(info.source) ?? []];
|
|
4333
|
+
if (read2.some((r) => r === path || stem(r) === stem(path)) || dataUnresolved.has(info.source) || dataTemplatePages.has(info.source)) out.push(prefix + info.out);
|
|
4334
|
+
}
|
|
4335
|
+
return out;
|
|
4336
|
+
},
|
|
4337
|
+
dataInTitles() {
|
|
4338
|
+
return dataInTitles;
|
|
4339
|
+
},
|
|
4340
|
+
invalidateData() {
|
|
4341
|
+
dataBy = /* @__PURE__ */ new Map();
|
|
4342
|
+
templatesBy = /* @__PURE__ */ new Map();
|
|
4343
|
+
dataDirHas = /* @__PURE__ */ new Map();
|
|
4148
4344
|
}
|
|
4149
4345
|
};
|
|
4150
4346
|
async function emitGenerated() {
|
|
@@ -4903,6 +5099,23 @@ async function openDevSession(input, o = {}) {
|
|
|
4903
5099
|
continue;
|
|
4904
5100
|
}
|
|
4905
5101
|
if (TOPIC_MAP.test(p)) continue;
|
|
5102
|
+
const dataDirs = (x) => [...new Set(x.unit.members.map((m) => dataDirOf(config, m.source)))].filter((d) => p.startsWith(`${d}/`));
|
|
5103
|
+
const dataUnits = [...units.values()].filter((x) => dataDirs(x).length);
|
|
5104
|
+
if (dataUnits.length) {
|
|
5105
|
+
for (const x of dataUnits) {
|
|
5106
|
+
if (x.handle.dataInTitles()) {
|
|
5107
|
+
lazy.add(x.unit.prefix);
|
|
5108
|
+
mark("unit");
|
|
5109
|
+
continue;
|
|
5110
|
+
}
|
|
5111
|
+
x.handle.invalidateData();
|
|
5112
|
+
for (const page of x.handle.dataReaders(p)) {
|
|
5113
|
+
stale.add(page);
|
|
5114
|
+
mark("page");
|
|
5115
|
+
}
|
|
5116
|
+
}
|
|
5117
|
+
continue;
|
|
5118
|
+
}
|
|
4906
5119
|
const u = unitOf(p);
|
|
4907
5120
|
if (!u) continue;
|
|
4908
5121
|
const content = input.files.get(p);
|
|
@@ -23,6 +23,7 @@ import { type FileRef, type TextRef, type TextStream } from "./detach.js";
|
|
|
23
23
|
import { type TemplateBudget } from "./preprocess.js";
|
|
24
24
|
import { ParseCache } from "@dogsbay/minja/browser";
|
|
25
25
|
import { ScanCache } from "./scan-cache.js";
|
|
26
|
+
import { DataCache, type DataLimits } from "@dogsbay/format-dogsbay-md/browser";
|
|
26
27
|
import { type RedirectInput, type RedirectRule } from "./site-files.js";
|
|
27
28
|
import { DiagramStore, type DiagramRenderer, type DiagramRenderers } from "./diagrams.js";
|
|
28
29
|
export type FileContent = string | Uint8Array | FileRef | TextRef;
|
|
@@ -75,6 +76,14 @@ export interface BuildInput {
|
|
|
75
76
|
sink?: (path: string, content: SinkContent) => void | Promise<void>;
|
|
76
77
|
/** Characters of include fragments to keep in memory between pages (default 8M). */
|
|
77
78
|
textCacheChars?: number;
|
|
79
|
+
/**
|
|
80
|
+
* Limits on the data files a build reads (docs/data-files.md in the dogsbay repo): per file, per
|
|
81
|
+
* build, values, depth, rows, columns, cell size. Default: the loader's (`DEFAULT_DATA_LIMITS`).
|
|
82
|
+
* An untrusted build never reads past `UNTRUSTED_DATA_LIMITS`, whatever is given here.
|
|
83
|
+
*/
|
|
84
|
+
dataLimits?: Partial<DataLimits>;
|
|
85
|
+
/** Bytes of parsed data files kept between pages and units (default 32 MB), weighed as the loader weighs them. */
|
|
86
|
+
dataCacheBytes?: number;
|
|
78
87
|
/** Called after each page, for progress reporting. */
|
|
79
88
|
onPage?: (done: number, total: number, unit: string) => void;
|
|
80
89
|
/**
|
|
@@ -141,11 +150,21 @@ export interface BuildInput {
|
|
|
141
150
|
/**
|
|
142
151
|
* Highlights code blocks off the main thread (the Node CLI passes a pool of
|
|
143
152
|
* worker threads). With it, pages are read and parsed a few ahead of their
|
|
144
|
-
* turn and their code blocks highlighted in the pool
|
|
145
|
-
*
|
|
146
|
-
*
|
|
153
|
+
* turn (`highlightAhead`) and their code blocks highlighted in the pool; each
|
|
154
|
+
* page renders from the blocks delivered for it, which then move into the
|
|
155
|
+
* highlight cache. Output is the same: a block the pool has not delivered is
|
|
156
|
+
* highlighted on the main thread as before.
|
|
147
157
|
*/
|
|
148
158
|
highlightPool?: HighlightPool;
|
|
159
|
+
/**
|
|
160
|
+
* How far ahead of rendering pages are prepared for the pool: at most `pages` pages (default 24; at least 1) and,
|
|
161
|
+
* beyond the next page, `chars` of their source plus the HTML delivered for them and not yet cached (default: no
|
|
162
|
+
* limit). Without a pool nothing is read ahead.
|
|
163
|
+
*/
|
|
164
|
+
highlightAhead?: {
|
|
165
|
+
pages?: number;
|
|
166
|
+
chars?: number;
|
|
167
|
+
};
|
|
149
168
|
/** Called once with the redirects the build writes (checked and resolved; site paths). */
|
|
150
169
|
onRedirects?: (rules: readonly RedirectRule[]) => void;
|
|
151
170
|
/**
|
|
@@ -161,9 +180,9 @@ export interface BuildInput {
|
|
|
161
180
|
export type SiteCssBuilder = (o: CssOptions) => Promise<string>;
|
|
162
181
|
/** The stylesheet builder of a build: the caller's, else Tailwind's. */
|
|
163
182
|
export declare function siteCssBuilder(input: BuildInput): Promise<SiteCssBuilder>;
|
|
164
|
-
/** Renders code blocks as `Highlighter.render` does, elsewhere; one result per request, in order. */
|
|
183
|
+
/** Renders code blocks as `Highlighter.render` does, elsewhere; one result per request, in order (null: not rendered there, so highlighted here). */
|
|
165
184
|
export interface HighlightPool {
|
|
166
|
-
render(blocks: CodeRequest[]): Promise<string[]>;
|
|
185
|
+
render(blocks: CodeRequest[]): Promise<(string | null)[]>;
|
|
167
186
|
}
|
|
168
187
|
/** A page as `BuildInput.onPageBuilt` sees it. */
|
|
169
188
|
export interface BuiltPage {
|
|
@@ -211,6 +230,8 @@ export interface SiteState {
|
|
|
211
230
|
parseCache: ParseCache | false;
|
|
212
231
|
/** Template time left for the build, or for a dev session's current request. */
|
|
213
232
|
templateBudget: TemplateBudget;
|
|
233
|
+
/** Parsed data files, by path and text (so an edited file is parsed again), shared by units and a dev session's rebuilds. */
|
|
234
|
+
dataCache: DataCache;
|
|
214
235
|
}
|
|
215
236
|
export interface BuildStats {
|
|
216
237
|
pages: number;
|
|
@@ -259,6 +280,9 @@ export interface BuildProblem {
|
|
|
259
280
|
/** The warning without its `[unit]` and file. */
|
|
260
281
|
message: string;
|
|
261
282
|
text: string;
|
|
283
|
+
/** For a data directive's problem in a data file: that file (a project path), and its 1-based line when known. */
|
|
284
|
+
dataFile?: string;
|
|
285
|
+
dataLine?: number;
|
|
262
286
|
}
|
|
263
287
|
/** Where a warning is, and its message when the text says more (a file first). */
|
|
264
288
|
type ProblemAt = Partial<Omit<BuildProblem, "code" | "text">>;
|
|
@@ -284,7 +308,7 @@ export interface PageDiagnostic {
|
|
|
284
308
|
* that one's lines; one only the templated text has has no `line`, and its message says the lines
|
|
285
309
|
* it names are the templated page's. Messages are detached (a session keeps them; some quote a line).
|
|
286
310
|
*/
|
|
287
|
-
export declare function linedDiagnostics(found: ParseDiagnostic[], raw: string, body: string, parsed: string): PageDiagnostic[];
|
|
311
|
+
export declare function linedDiagnostics(found: ParseDiagnostic[], raw: string, body: string, parsed: string, after?: string): PageDiagnostic[];
|
|
288
312
|
/** A file's text, read now if it is lazy. */
|
|
289
313
|
export declare function readText(c: FileContent): Promise<string>;
|
|
290
314
|
/**
|
|
@@ -319,6 +343,16 @@ export interface UnitHandle {
|
|
|
319
343
|
forget(path: string): void;
|
|
320
344
|
/** Whether a page of the unit has included this file (so its includers must render again). */
|
|
321
345
|
included(path: string): boolean;
|
|
346
|
+
/**
|
|
347
|
+
* The pages to render again when a data file (a project path) changes: those whose directives or
|
|
348
|
+
* template read it, those whose data directives were left in place, and template pages that use
|
|
349
|
+
* `data`. Empty when the unit does not read its folder.
|
|
350
|
+
*/
|
|
351
|
+
dataReaders(path: string): string[];
|
|
352
|
+
/** Whether a template page's title or description uses `data` (resolved in pass 1: a change needs the unit built again). */
|
|
353
|
+
dataInTitles(): boolean;
|
|
354
|
+
/** Forget what the unit loaded from data files: readers, template pages and their values, which folders had files. */
|
|
355
|
+
invalidateData(): void;
|
|
322
356
|
/** The project files the unit's nav was read from: `nav:` files, topic maps and every map they include. */
|
|
323
357
|
navInputs: string[];
|
|
324
358
|
}
|
|
@@ -43,6 +43,12 @@ export interface SourceDef {
|
|
|
43
43
|
* source's root, else its folders.
|
|
44
44
|
*/
|
|
45
45
|
nav?: string;
|
|
46
|
+
/**
|
|
47
|
+
* The source's own data folder (`data:` on the source, docs/data-files.md in the dogsbay repo),
|
|
48
|
+
* project-relative: a versioned site whose versions need different data gives each its own.
|
|
49
|
+
* Unset: the site's (`SiteConfig.dataDir`).
|
|
50
|
+
*/
|
|
51
|
+
data?: string;
|
|
46
52
|
}
|
|
47
53
|
export type SearchEngine = "builtin" | "pagefind" | "both";
|
|
48
54
|
export interface SiteConfig {
|
|
@@ -139,6 +145,14 @@ export interface SiteConfig {
|
|
|
139
145
|
* default (BuildInput.defaultHtml), "keep" for local builds, "sanitize" in the Worker.
|
|
140
146
|
*/
|
|
141
147
|
html?: "keep" | "sanitize";
|
|
148
|
+
/**
|
|
149
|
+
* The data folder (content.data, shared with the Astro pipeline): YAML, JSON, CSV and TSV files
|
|
150
|
+
* pages read by name (`::data-table`, `::data-cards`, `data.<name>` in template pages).
|
|
151
|
+
* Project-relative; "data" (beside dogsbay.config.yml) unless set. `dataDirSet`: named in the
|
|
152
|
+
* config, so a folder with no files is reported.
|
|
153
|
+
*/
|
|
154
|
+
dataDir: string;
|
|
155
|
+
dataDirSet: boolean;
|
|
142
156
|
/** Click-to-zoom images, from `plugins: [@dogsbay/plugin-image-zoom]` with its options. */
|
|
143
157
|
imageZoom?: {
|
|
144
158
|
background: string;
|
|
@@ -278,9 +292,12 @@ export declare function projectPath(unit: BuildUnit, path: string): string;
|
|
|
278
292
|
export declare const TOPIC_MAP: RegExp;
|
|
279
293
|
/**
|
|
280
294
|
* The project folders and files a build reads: every source's root, the theme folder, extra
|
|
281
|
-
* stylesheets,
|
|
295
|
+
* stylesheets, the folder of each source's own nav file (a topic map's includes sit beside it), and
|
|
296
|
+
* the data folders (the site's, and any source's own; one that does not exist is not read).
|
|
282
297
|
* [""] when a source is the whole project. What the CLI reads from disk, and a dev session watches.
|
|
283
298
|
*/
|
|
284
299
|
export declare function projectRoots(cfg: SiteConfig): string[];
|
|
300
|
+
/** The data folder a source's pages read: its own `data:`, else the site's. */
|
|
301
|
+
export declare function dataDirOf(cfg: SiteConfig, s: SourceDef): string;
|
|
285
302
|
/** A source's content root as a path prefix: "" for ".", else "content/". */
|
|
286
303
|
export declare function sourceRoot(s: SourceDef): string;
|
|
@@ -10,6 +10,10 @@
|
|
|
10
10
|
* its pages (`rebuildUnitLazy`), its old output is dropped, and all its
|
|
11
11
|
* pages are stale. What pass 1 read from each unchanged page is kept
|
|
12
12
|
* between rebuilds (src/core/scan-cache.ts), so this is cheap;
|
|
13
|
+
* - a data file (in the site's data folder or a source's own): the pages
|
|
14
|
+
* that read it (or a file of the same name), have data directives left
|
|
15
|
+
* in place, or are template pages of that folder are stale; a unit whose
|
|
16
|
+
* template titles use `data` is built again;
|
|
13
17
|
* - a topic map, a map it includes or a source's `nav:` file: the units
|
|
14
18
|
* that read it are built again, as above;
|
|
15
19
|
* - an asset: its entry is replaced;
|
|
@@ -14,7 +14,7 @@ export type DiagramResult = {
|
|
|
14
14
|
};
|
|
15
15
|
export interface DiagramRenderer {
|
|
16
16
|
render(batch: DiagramRequest[]): Promise<DiagramResult[]>;
|
|
17
|
-
/** Calls allowed per build
|
|
17
|
+
/** Calls allowed per build. Default: no limit (the Worker's renderer, on one session, has none). */
|
|
18
18
|
maxCalls?: number;
|
|
19
19
|
/** Most diagrams per call; larger batches are split (each part is a call). Default: no limit. */
|
|
20
20
|
maxBatch?: number;
|
|
@@ -55,10 +55,10 @@ export declare function fencedDiagrams(src: string): string[];
|
|
|
55
55
|
*
|
|
56
56
|
* The SVGs the current unit's built pages use are pinned until `endUnit()`:
|
|
57
57
|
* evicting one would cost a renderer call to get it back (the first deployed
|
|
58
|
-
* test spent 20 of
|
|
59
|
-
* `maxPinnedChars`, so memory stays bounded however many
|
|
60
|
-
* has; beyond it, SVGs are evicted and pages render them
|
|
61
|
-
*
|
|
58
|
+
* test, when each call was a Worker call, spent 20 of its 30 that way).
|
|
59
|
+
* Pinning stops at `maxPinnedChars`, so memory stays bounded however many
|
|
60
|
+
* diagrams a unit has; beyond it, SVGs are evicted and pages render them
|
|
61
|
+
* again, within `maxCalls` if the renderer has one. Unpinned SVGs are kept up to `maxChars` for later
|
|
62
62
|
* units, least recently used first out.
|
|
63
63
|
*/
|
|
64
64
|
export declare class DiagramStore {
|
|
@@ -14,6 +14,16 @@ export interface PreprocessResult {
|
|
|
14
14
|
error?: string;
|
|
15
15
|
/** Not rendered: the build's template time budget was spent (counted in `TemplateBudget.skipped`). */
|
|
16
16
|
skipped?: boolean;
|
|
17
|
+
/**
|
|
18
|
+
* Tags content mode does not know (`{% for %}`, `{% macro %}`), kept as written: minja's warnings
|
|
19
|
+
* (outside code, which is masked). `line` is in `text` when `include` is unset.
|
|
20
|
+
*/
|
|
21
|
+
warnings?: {
|
|
22
|
+
tag: string;
|
|
23
|
+
line: number;
|
|
24
|
+
include?: string;
|
|
25
|
+
variable?: string;
|
|
26
|
+
}[];
|
|
17
27
|
}
|
|
18
28
|
/**
|
|
19
29
|
* Template time for a whole build (plans/worker-scale.md stage 3): each render gets the shorter of
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@dogsbay/ssg",
|
|
3
|
-
"version": "0.2.0-beta.
|
|
3
|
+
"version": "0.2.0-beta.138",
|
|
4
4
|
"type": "module",
|
|
5
5
|
"description": "Documentation static site generator for Dogsbay-MD that builds inside Cloudflare Workers and publishes sites that work offline",
|
|
6
6
|
"license": "Apache-2.0",
|
|
@@ -33,8 +33,10 @@
|
|
|
33
33
|
"test": "npm run build:client && vitest run",
|
|
34
34
|
"test:browser": "playwright test",
|
|
35
35
|
"check": "npm run build:client && tsc --noEmit && tsc --noEmit -p src/worker/tsconfig.json",
|
|
36
|
-
"dev:worker": "npm run build:client && wrangler dev -c wrangler.jsonc -c wrangler.diagrams.jsonc --port 8798",
|
|
36
|
+
"dev:worker": "npm run build:client && wrangler dev -c wrangler.jsonc -c wrangler.diagrams.jsonc -c wrangler.highlight.jsonc --port 8798",
|
|
37
37
|
"deploy": "npm run build:client && wrangler deploy",
|
|
38
|
+
"deploy:diagrams": "npm run build:client && wrangler deploy -c wrangler.diagrams.jsonc",
|
|
39
|
+
"deploy:highlight": "npm run build:client && wrangler deploy -c wrangler.highlight.jsonc",
|
|
38
40
|
"test:worker": "npm run build:client && node scripts/test-worker.mjs",
|
|
39
41
|
"build:package": "npm run build:client && node scripts/build-package.mjs",
|
|
40
42
|
"prepublishOnly": "npm run build:package"
|
|
@@ -47,38 +49,38 @@
|
|
|
47
49
|
"tailwindcss": "^4.3.2"
|
|
48
50
|
},
|
|
49
51
|
"peerDependencies": {
|
|
50
|
-
"@dogsbay/elements": "0.2.0-beta.
|
|
51
|
-
"@dogsbay/format-dogsbay-md": "0.2.0-beta.
|
|
52
|
-
"@dogsbay/format-openapi": "0.2.0-beta.
|
|
53
|
-
"@dogsbay/format-topicmap": "0.2.0-beta.
|
|
54
|
-
"@dogsbay/highlight": "0.2.0-beta.
|
|
55
|
-
"@dogsbay/minja": "0.2.0-beta.
|
|
56
|
-
"@dogsbay/primitives": "0.2.0-beta.
|
|
57
|
-
"@dogsbay/render-html": "0.2.0-beta.
|
|
58
|
-
"@dogsbay/search": "0.2.0-beta.
|
|
59
|
-
"@dogsbay/serialize-core": "0.2.0-beta.
|
|
60
|
-
"@dogsbay/site-check": "0.2.0-beta.
|
|
61
|
-
"@dogsbay/site-files": "0.2.0-beta.
|
|
62
|
-
"@dogsbay/types": "0.2.0-beta.
|
|
52
|
+
"@dogsbay/elements": "0.2.0-beta.138",
|
|
53
|
+
"@dogsbay/format-dogsbay-md": "0.2.0-beta.138",
|
|
54
|
+
"@dogsbay/format-openapi": "0.2.0-beta.138",
|
|
55
|
+
"@dogsbay/format-topicmap": "0.2.0-beta.138",
|
|
56
|
+
"@dogsbay/highlight": "0.2.0-beta.138",
|
|
57
|
+
"@dogsbay/minja": "0.2.0-beta.138",
|
|
58
|
+
"@dogsbay/primitives": "0.2.0-beta.138",
|
|
59
|
+
"@dogsbay/render-html": "0.2.0-beta.138",
|
|
60
|
+
"@dogsbay/search": "0.2.0-beta.138",
|
|
61
|
+
"@dogsbay/serialize-core": "0.2.0-beta.138",
|
|
62
|
+
"@dogsbay/site-check": "0.2.0-beta.138",
|
|
63
|
+
"@dogsbay/site-files": "0.2.0-beta.138",
|
|
64
|
+
"@dogsbay/types": "0.2.0-beta.138"
|
|
63
65
|
},
|
|
64
66
|
"devDependencies": {
|
|
65
67
|
"@axe-core/playwright": "^4.12.1",
|
|
66
68
|
"@cloudflare/workers-types": "^5.20260921.1",
|
|
67
|
-
"@dogsbay/elements": "0.2.0-beta.
|
|
68
|
-
"@dogsbay/format-dogsbay-md": "0.2.0-beta.
|
|
69
|
-
"@dogsbay/format-openapi": "0.2.0-beta.
|
|
70
|
-
"@dogsbay/format-topicmap": "0.2.0-beta.
|
|
71
|
-
"@dogsbay/highlight": "0.2.0-beta.
|
|
72
|
-
"@dogsbay/icons": "0.2.0-beta.
|
|
73
|
-
"@dogsbay/minja": "0.2.0-beta.
|
|
74
|
-
"@dogsbay/primitives": "0.2.0-beta.
|
|
75
|
-
"@dogsbay/render-html": "0.2.0-beta.
|
|
76
|
-
"@dogsbay/search": "0.2.0-beta.
|
|
77
|
-
"@dogsbay/serialize-core": "0.2.0-beta.
|
|
78
|
-
"@dogsbay/site-check": "0.2.0-beta.
|
|
79
|
-
"@dogsbay/site-files": "0.2.0-beta.
|
|
80
|
-
"@dogsbay/types": "0.2.0-beta.
|
|
81
|
-
"@dogsbay/ui": "0.2.0-beta.
|
|
69
|
+
"@dogsbay/elements": "0.2.0-beta.138",
|
|
70
|
+
"@dogsbay/format-dogsbay-md": "0.2.0-beta.138",
|
|
71
|
+
"@dogsbay/format-openapi": "0.2.0-beta.138",
|
|
72
|
+
"@dogsbay/format-topicmap": "0.2.0-beta.138",
|
|
73
|
+
"@dogsbay/highlight": "0.2.0-beta.138",
|
|
74
|
+
"@dogsbay/icons": "0.2.0-beta.138",
|
|
75
|
+
"@dogsbay/minja": "0.2.0-beta.138",
|
|
76
|
+
"@dogsbay/primitives": "0.2.0-beta.138",
|
|
77
|
+
"@dogsbay/render-html": "0.2.0-beta.138",
|
|
78
|
+
"@dogsbay/search": "0.2.0-beta.138",
|
|
79
|
+
"@dogsbay/serialize-core": "0.2.0-beta.138",
|
|
80
|
+
"@dogsbay/site-check": "0.2.0-beta.138",
|
|
81
|
+
"@dogsbay/site-files": "0.2.0-beta.138",
|
|
82
|
+
"@dogsbay/types": "0.2.0-beta.138",
|
|
83
|
+
"@dogsbay/ui": "0.2.0-beta.138",
|
|
82
84
|
"@mermanjs/web": "0.7.0",
|
|
83
85
|
"@playwright/test": "1.61.1",
|
|
84
86
|
"@types/node": "^22.10.0",
|