@heroiclands/package-build 24.0.0 → 24.0.2

package/engine/script-font-delivery.mjs

@@ -13,9 +13,9 @@
 
 /**
  * **Delivering the installed faces to each target.** A consumer that installs
- * `@heroiclands/fonts` and sets text in one of its faces with `font=` gets the
- * fonts on the site, in the Foundry module and in its books; without both,
- * nothing ships and nothing links to it.
+ * `@heroiclands/fonts` and sets text in one of its faces, with `font=` or by a
+ * language note's `data.writtenIn`, gets the fonts on the site, in the Foundry
+ * module and in its books; without both, nothing ships and nothing links to it.
  *
  * - **Site and Foundry** receive `fonts.css`, listing only the WOFF2 sources,
  *   the WOFF2 files and `LICENSE`.
@@ -27,9 +27,12 @@
 import fs from "node:fs";
 import path from "node:path";
 
+import YAML from "yaml";
+
 import { baseStyleLayer } from "./base-styles.mjs";
 import { scanRuns } from "./content-scripts.mjs";
 import { FONTS_MANIFEST, resolveScriptFonts } from "./script-fonts.mjs";
+import { LANGUAGE_SUBTYPE } from "./word-terms.mjs";
 
 /**
  * Where the files land in the staged Foundry module, relative to its root.
@@ -65,10 +68,37 @@ const LICENSE = "LICENSE";
 /** The faces each configuration's notes set text in, found once. */
 const usedFaces = new WeakMap();
 
+/** A note's frontmatter block. */
+const FRONTMATTER = /^\uFEFF?---\r?\n([\s\S]*?)\r?\n---(?:\r?\n|$)/;
+
+/**
+ * The face a language note names in `data.writtenIn`, where it is installed.
+ *
+ * @param {string} source - The note's file.
+ * @param {import("./script-fonts.mjs").ScriptFonts} installed - The fonts package.
+ * @returns {string|undefined} The face's slug.
+ */
+function writtenInFace(source, installed) {
+    const yaml = FRONTMATTER.exec(source)?.[1];
+    if (yaml === undefined) return undefined;
+    let fm;
+    try {
+        fm = YAML.parse(yaml);
+    } catch {
+        return undefined;
+    }
+    if (fm?.type !== "skill" || String(fm.subType ?? "").toLowerCase() !== LANGUAGE_SUBTYPE)
+        return undefined;
+    const font = fm.data?.writtenIn?.font;
+    return typeof font === "string" && installed.faces.has(font) ? font : undefined;
+}
+
 /**
- * The faces the package's notes set text in with `font=`, read from the
- * notes themselves: every Markdown file under the content tree that writes
- * `font=` is scanned for its runs.
+ * The faces the package's notes set text in, read from the notes
+ * themselves: every Markdown file under the content tree that writes `font=`
+ * is scanned for its runs, and the face every language note names in
+ * `data.writtenIn` counts, since a run marked with the language alone is set
+ * in it.
  *
  * @param {DeliveryConfig} config - The resolved configuration.
  * @param {import("./script-fonts.mjs").ScriptFonts} installed - The fonts package.
@@ -90,10 +120,13 @@ export function facesUsed(config, installed) {
             if (entry.isDirectory()) walk(full);
             else if (entry.isFile() && entry.name.endsWith(".md")) {
                 const source = fs.readFileSync(full, "utf8");
+                if (/^\s*writtenIn:/m.test(source)) {
+                    const face = writtenInFace(source, installed);
+                    if (face) used.add(face);
+                }
                 if (!/\bfont=/.test(source)) continue;
-                const body = source.replace(
-                    /^\uFEFF?---\r?\n[\s\S]*?\r?\n---(?:\r?\n|$)/,
-                    (frontmatter) => frontmatter.replace(/[^\r\n]/g, " "),
+                const body = source.replace(FRONTMATTER, (frontmatter) =>
+                    frontmatter.replace(/[^\r\n]/g, " "),
                 );
                 const ctx = { languages: config.languages ?? {}, faces: installed.faces };
                 for (const run of scanRuns(body, ctx).runs)

package/engine/script-fonts.mjs

@@ -68,6 +68,9 @@ export const FONTS_MANIFEST = "fonts.json";
  * @property {string} direction - `ltr` or `rtl`.
  * @property {string[]} features - OpenType features it turns on.
  * @property {ReadonlySet<number>} draws - Every code point it draws.
+ * @property {string|undefined} file - The absolute path of its TrueType file,
+ *   which the book draws a `scroll` fence's outlines from; `undefined` when the
+ *   manifest lists none.
  */
 
 /**
@@ -164,6 +167,10 @@ export function resolveScriptFonts(rootDir) {
                 direction: entry.direction === "rtl" ? "rtl" : "ltr",
                 features: Object.freeze([...(entry.features ?? [])]),
                 draws,
+                file:
+                    typeof entry.files?.ttf === "string" ?
+                        path.join(root, entry.files.ttf)
+                    :   undefined,
             }),
         );
     }

package/engine/site-build.mjs

@@ -99,6 +99,7 @@ import { checkHomepageCount, homepageFrontmatter, homepageTitle, isHomepage } fr
 import { publishesContentPages, siteBase } from "../content-config.mjs";
 import { seeAlsoPages } from "./see-also.mjs";
 import { homepageLinkTargets, relatedPages } from "./related-pages.mjs";
+import { GENERATED_MARK } from "./site-reach.mjs";
 import { drawSiteMaps, inlineDrawingFigure } from "./site-maps.mjs";
 import { generatedDrawing } from "./generated-sections.mjs";
 import { IMAGE_PATTERN } from "./content-images.mjs";
@@ -784,7 +785,13 @@ export function renderSitePage(
     const tableErrors = expanded.findings.tables;
     const expressionErrors = expanded.findings.expressions;
     const wikiErrors = expanded.findings.links;
-    const resolvedBody = expanded.markdown;
+    // The author's text and the sections the build generated after it are
+    // told apart by a comment, so the site writer can leave the generated
+    // links out of what makes a page reachable.
+    const resolvedBody =
+        expanded.generated ?
+            `${expanded.authored}\n\n${GENERATED_MARK}\n\n${expanded.generated}`
+        :   expanded.markdown;
     for (const error of expanded.findings.captions)
         captionErrors.push({
             file: page.file,

package/engine/site-reach.mjs

@@ -0,0 +1,213 @@
+/*
+ * This file is part of the Song of Heroic Lands (SoHL) system for Foundry VTT.
+ * Copyright (c) 2024-2026 Tom Rodriguez ("Toasty") — <toasty@heroiclands.org>
+ *
+ * This work is licensed under the GNU General Public License v3.0 (GPLv3).
+ * You may copy, modify, and distribute it under the terms of that license.
+ *
+ * For full terms, see the LICENSE.md file in the project root or visit:
+ * https://www.gnu.org/licenses/gpl-3.0.html
+ *
+ * SPDX-License-Identifier: GPL-3.0-or-later
+ */
+
+/**
+ * **The pages a reader can reach.** Every note is written as a page, but what
+ * a reader finds is what the homepage leads to: the sitemap and the search
+ * index cover the pages reachable from the homepage by the links the rendered
+ * HTML carries, and no others. An unlinked page is still served at its
+ * address, so an author reads a page in place before linking it to publish it.
+ *
+ * Reachability is read from the rendered pages, never from the source notes:
+ * the `<a>` and `<link>` targets that lie inside the site's base path, the
+ * menu and the chrome included. A link that lands on a feed makes the pages
+ * the feed lists reachable too, because a feed is a list of links.
+ *
+ * @module
+ */
+
+/** A URL with a scheme, or a protocol-relative one: not a path inside the site. */
+const EXTERNAL = /^(?:[a-z][a-z0-9+.-]*:|\/\/)/i;
+
+/** An `<a>` or `<link>` and its `href`, quoted either way or bare. */
+const LINK = /<(?:a|link)\b[^>]*?\shref=(?:"([^"]*)"|'([^']*)'|([^\s>]+))/gi;
+
+/**
+ * The internal targets a page links.
+ *
+ * @param {string} html - A rendered page.
+ * @param {string} from - The page's own address, beginning with `/`.
+ * @param {string} base - The site's base path, beginning and ending with `/`.
+ * @returns {string[]} Absolute paths inside the base, without fragment or query.
+ */
+export function internalLinks(html, from, base) {
+    const found = [];
+    for (const match of html.matchAll(LINK)) {
+        const href = (match[1] ?? match[2] ?? match[3] ?? "").trim().replace(/&amp;/g, "&");
+        if (!href || href.startsWith("#") || EXTERNAL.test(href)) continue;
+        let pathname;
+        try {
+            pathname = new URL(href, `http://site.invalid${from}`).pathname;
+        } catch {
+            continue;
+        }
+        if (pathname.startsWith(base)) found.push(decodeURI(pathname));
+    }
+    return found;
+}
+
+/**
+ * The address a link to a file names a page by: `…/index.html` is its
+ * directory, and a bare directory ends in a slash.
+ *
+ * @param {string} pathname - An absolute path.
+ * @returns {string} The page address it would be keyed by.
+ */
+export function pageAddress(pathname) {
+    if (pathname.endsWith("/index.html")) return pathname.slice(0, -"index.html".length);
+    return pathname.endsWith("/") || /\.[a-z0-9]+$/i.test(pathname) ? pathname : `${pathname}/`;
+}
+
+/**
+ * The pages reachable from the homepage.
+ *
+ * @param {object} options - The inputs.
+ * @param {string} options.base - The site's base path, beginning and ending
+ *   with `/`; the homepage is at the base.
+ * @param {(url: string) => string|undefined} options.html - A page's rendered
+ *   HTML by its address, or `undefined` where the site has no such page.
+ * @param {(pathname: string) => readonly string[]} [options.expand] - The page
+ *   addresses a linked non-page file, such as a feed, lists.
+ * @returns {Set<string>} The address of every reachable page, the homepage
+ *   included.
+ */
+export function reachablePages({ base, html, expand = () => [] }) {
+    /** @type {Set<string>} */
+    const reached = new Set();
+    const queue = [base];
+    const seenFiles = new Set();
+    while (queue.length) {
+        const url = queue.pop();
+        if (reached.has(url)) continue;
+        const body = html(url);
+        if (body === undefined) continue;
+        reached.add(url);
+        for (const target of internalLinks(body, url, base)) {
+            const address = pageAddress(target);
+            if (html(address) !== undefined) {
+                queue.push(address);
+            } else if (!seenFiles.has(target)) {
+                seenFiles.add(target);
+                for (const listed of expand(target)) queue.push(pageAddress(listed));
+            }
+        }
+    }
+    return reached;
+}
+
+/**
+ * The page addresses an Atom feed lists, as paths.
+ *
+ * @param {string} xml - The feed.
+ * @returns {string[]} The `href` of every `<link>` that is an absolute URL,
+ *   as its path.
+ */
+export function feedPaths(xml) {
+    const paths = [];
+    for (const match of xml.matchAll(/<link\b[^>]*?\shref="([^"]*)"/g)) {
+        try {
+            paths.push(new URL(match[1].replace(/&amp;/g, "&")).pathname);
+        } catch {
+            // A relative or malformed link names no page.
+        }
+    }
+    return paths;
+}
+
+/**
+ * The comment that separates a page's authored text from the sections the
+ * build generated after it, in the HTML the renderer writes.
+ *
+ * @type {string}
+ */
+export const GENERATED_MARK = "<!--generated-sections-->";
+
+/**
+ * A page body split at {@link GENERATED_MARK}.
+ *
+ * @param {string} html - A page body.
+ * @returns {{authored: string, generated: string}} The author's text, and the
+ *   generated sections after it, `""` where there are none.
+ */
+export function splitContent(html) {
+    const text = String(html ?? "");
+    const at = text.indexOf(GENERATED_MARK);
+    if (at < 0) return { authored: text, generated: "" };
+    return { authored: text.slice(0, at), generated: text.slice(at + GENERATED_MARK.length) };
+}
+
+/** An `<a>` element with an `href`, whole. */
+const ANCHOR = /<a\b[^>]*?\shref=(?:"([^"]*)"|'([^']*)'|([^\s>]+))[^>]*>[\s\S]*?<\/a>/gi;
+
+/**
+ * Generated sections with the entries that lead outside a set of pages taken
+ * out.
+ *
+ * An entry is a list item, a table row, or a link in a paragraph's comma
+ * separated run; a paragraph, list or table left with no entry goes, and so
+ * does a section left with nothing under its heading. A link to anything that
+ * is not a page of the site is kept, and so is text that was never a link.
+ *
+ * @param {string} html - The generated sections, rendered.
+ * @param {(href: string) => boolean} outside - Whether an `href` names a page
+ *   of the site that is not reachable.
+ * @returns {string} The sections, or `""` when nothing is left.
+ */
+export function pruneGenerated(html, outside) {
+    const dead = (anchor) => {
+        const href = new RegExp(ANCHOR.source, "i").exec(anchor);
+        const value = href ? (href[1] ?? href[2] ?? href[3] ?? "").replace(/&amp;/g, "&") : "";
+        return value !== "" && outside(value);
+    };
+    const holdsDead = (fragment) => [...fragment.matchAll(ANCHOR)].some((match) => dead(match[0]));
+
+    // Each section runs from its heading to the next one.
+    const parts = String(html ?? "").split(/(?=<h1\b)/);
+    const kept = parts.map((part) => {
+        if (!holdsDead(part)) return part;
+        let out = part
+            .replace(/<li\b[^>]*>(?:(?!<\/li>)[\s\S])*<\/li>\s*/g, (item) =>
+                holdsDead(item) ? "" : item,
+            )
+            .replace(/<tr\b[^>]*>(?:(?!<\/tr>)[\s\S])*<\/tr>\s*/g, (row) =>
+                holdsDead(row) ? "" : row,
+            )
+            .replace(/<p\b[^>]*>(?:(?!<\/p>)[\s\S])*<\/p>\s*/g, (paragraph) => {
+                if (!holdsDead(paragraph)) return paragraph;
+                const trimmed = paragraph.replace(ANCHOR, (anchor) =>
+                    dead(anchor) ? "\u0000" : anchor,
+                );
+                const entries = trimmed
+                    .replace(/<\/?p\b[^>]*>/g, "")
+                    .replace(/^\s*<strong>[^<]*<\/strong>/, "")
+                    .replace(/\u0000/g, "")
+                    .replace(/[\s,]+/g, " ")
+                    .replace(/<[^>]*>/g, "")
+                    .trim();
+                if (!entries) return "";
+                return trimmed
+                    .replace(/\u0000,\s*/g, "")
+                    .replace(/,\s*\u0000/g, "")
+                    .replace(/\u0000/g, "");
+            });
+        out = out
+            .replace(/<(ul|ol)\b[^>]*>\s*<\/\1>\s*/g, "")
+            .replace(/<tbody\b[^>]*>\s*<\/tbody>\s*/g, "")
+            .replace(/<table\b[^>]*>(?:(?!<tr\b)[\s\S])*<\/table>\s*/g, (table) =>
+                /<tr\b/.test(table) ? table : "",
+            );
+        const rest = out.replace(/^<h1\b[^>]*>[\s\S]*?<\/h1>/, "").replace(/<!--[\s\S]*?-->/g, "");
+        return rest.trim() === "" ? "" : out;
+    });
+    return kept.join("");
+}

package/engine/site-render.mjs

@@ -50,6 +50,12 @@
  * | `feed: {section, title}` | `<section>/atom.xml`, the section's pages, newest first |
  * | `sitemaps: [...]` | `sitemap-index.xml`, naming this site's sitemap and each one listed, and `robots.txt` naming them all |
  *
+ * Every page is written, but `sitemap.xml` lists only the pages reachable from
+ * the homepage by the links the rendered pages carry; see
+ * {@link module:engine/site-reach}. A feed is a link, so the pages it lists
+ * are reachable when a page links the feed, and a section or tag page lists
+ * pages it links to the same way.
+ *
  * A sitemap index and `robots.txt` belong at the root of an origin, so a site
  * served under a path refuses `sitemaps`.
  *
@@ -76,6 +82,7 @@ import {
 import { buildSite, gatesFailed } from "./site-build.mjs";
 import { readCachedNavigation, siteSettings } from "./site-config.mjs";
 import { atomFeed, robotsTxt, sitemapIndexXml, sitemapXml } from "./site-files.mjs";
+import { pageAddress, pruneGenerated, reachablePages, splitContent } from "./site-reach.mjs";
 import { prepareTreeSqlTables } from "./sql-tables.mjs";
 import { placeGeography, searchMetadata } from "./site-search.mjs";
 import {
@@ -298,8 +305,9 @@ export function writeThemeFiles(out, staticDir = SITE_STATIC_DIR) {
  * @param {string} [options.staticDir] - The theme's own files, copied first.
  * @param {ReadonlySet<string>} [options.only] - The addresses of the pages to
  *   draw again into a site already written. Nothing else is touched.
- * @returns {{files: string[], pages: string[], findings: object[]}} Every file
- *   written, the address of every page, and the layouts' findings.
+ * @returns {{files: string[], pages: string[], reachable: string[],
+ *   findings: object[]}} Every file written, the address of every page, the
+ *   address of each page reachable from the homepage, and the layouts' findings.
  * @throws {Error} When a page's address is outside the site, or a site served
  *   under a path asks for a sitemap index.
  */
@@ -311,32 +319,95 @@ export function writeSite({ site, pages, out, report, staticDir = SITE_STATIC_DI
     const tree = siteTree(site, pages);
     const published = new Set(tree.map((page) => page.url));
     const feedUrl = site.feed ? relUrl(site, `${site.feed.section}/atom.xml`) : "";
-    const theme = createSiteTheme({
-        site: {
-            ...site,
-            feed:
-                site.feed ?
-                    {
-                        title: site.feed.title ?? site.title,
-                        url: feedUrl,
-                        section: text(site.feed.section),
-                    }
-                :   undefined,
-            pages: tree
-                .filter((page) => page.kind === "page")
-                .map((page) => ({
-                    url: page.url,
-                    title: page.title,
-                    section: text(page.section),
-                    type: page.params?.type,
-                    subType: page.params?.subType,
-                    date: page.date ?? page.params?.date,
-                    content: page.content,
-                })),
-            pageExists: (url) => published.has(url) || published.has(`${url.replace(/\/?$/, "/")}`),
-        },
-        report,
+    const base = basePath(site);
+    const origin = (url) => new URL(url, site.baseURL).href;
+
+    // The address of the page a URL names, or undefined where it names none of
+    // this site's.
+    const pageOfUrl = (url) => {
+        let pathname;
+        try {
+            const resolved = new URL(String(url), origin(base));
+            if (resolved.origin !== new URL(site.baseURL).origin) return undefined;
+            pathname = decodeURI(resolved.pathname);
+        } catch {
+            return undefined;
+        }
+        const address = pageAddress(pathname);
+        return published.has(address) ? address : undefined;
+    };
+
+    // A page is drawn twice. The first draw has no generated link to a page,
+    // and the pages it links are the reachable ones; the second draws the
+    // generated sections again with the links that stay inside that set.
+    const makeTheme = ({ visible, contentOf, report: sink }) =>
+        createSiteTheme({
+            site: {
+                ...site,
+                visible,
+                feed:
+                    site.feed ?
+                        {
+                            title: site.feed.title ?? site.title,
+                            url: feedUrl,
+                            section: text(site.feed.section),
+                        }
+                    :   undefined,
+                pages: tree
+                    .filter((page) => page.kind === "page")
+                    .map((page) => ({
+                        url: page.url,
+                        title: page.title,
+                        section: text(page.section),
+                        type: page.params?.type,
+                        subType: page.params?.subType,
+                        date: page.date ?? page.params?.date,
+                        content: contentOf(page),
+                    })),
+                pageExists: (url) =>
+                    published.has(url) || published.has(`${url.replace(/\/?$/, "/")}`),
+            },
+            report: sink,
+        });
+    const authoredOf = (page) => splitContent(page.content).authored;
+    const first = makeTheme({
+        visible: (url) => pageOfUrl(url) === undefined,
+        contentOf: authoredOf,
+    });
+    const firstPass = new Map();
+    for (const page of tree)
+        firstPass.set(
+            page.url,
+            first.render({
+                ...page,
+                content: page.content === undefined ? undefined : authoredOf(page),
+                permalink: origin(page.url),
+            }),
+        );
+
+    const section = site.feed ? text(site.feed.section) : "";
+    const time = (page) => new Date(text(page.date ?? page.params?.date)).getTime() || 0;
+    const posts =
+        site.feed ?
+            tree
+                .filter((page) => page.kind === "page" && text(page.section) === section)
+                .sort((a, b) => time(b) - time(a))
+        :   [];
+    const reachable = reachablePages({
+        base,
+        html: (url) => firstPass.get(url),
+        expand: (target) => (site.feed && target === feedUrl ? posts.map((post) => post.url) : []),
     });
+    const outside = (url) => {
+        const address = pageOfUrl(url);
+        return address !== undefined && !reachable.has(address);
+    };
+    const contentOf = (page) => {
+        if (page.content === undefined) return page.content;
+        const { authored, generated } = splitContent(page.content);
+        return authored + pruneGenerated(generated, outside);
+    };
+    const theme = makeTheme({ visible: (url) => !outside(url), contentOf, report });
 
     const files = [];
     const write = (file, content) => {
@@ -344,15 +415,22 @@ export function writeSite({ site, pages, out, report, staticDir = SITE_STATIC_DI
         fs.writeFileSync(file, content);
         files.push(file);
     };
-    const origin = (url) => new URL(url, site.baseURL).href;
-    const draw = (page) =>
-        write(
-            pageFile(site, out, page.url),
-            minifyHtml(theme.render({ ...page, permalink: origin(page.url) })),
+    const draw = (page) => {
+        const html = minifyHtml(
+            theme.render({ ...page, content: contentOf(page), permalink: origin(page.url) }),
         );
+        write(pageFile(site, out, page.url), html);
+    };
+    const result = (extra = {}) => ({
+        files,
+        pages: tree.map((page) => page.url),
+        reachable: [...reachable],
+        findings: theme.findings,
+        ...extra,
+    });
     if (only) {
         for (const page of tree) if (only.has(page.url)) draw(page);
-        return { files, pages: tree.map((page) => page.url), findings: theme.findings };
+        return result();
     }
 
     fs.rmSync(out, { recursive: true, force: true });
@@ -373,19 +451,16 @@ export function writeSite({ site, pages, out, report, staticDir = SITE_STATIC_DI
     write(
         path.join(out, "sitemap.xml"),
         sitemapXml(
-            tree.map((page) => ({
-                loc: origin(page.url),
-                lastmod: page.params?.lastmod ?? page.date ?? page.params?.date,
-            })),
+            tree
+                .filter((page) => reachable.has(page.url))
+                .map((page) => ({
+                    loc: origin(page.url),
+                    lastmod: page.params?.lastmod ?? page.date ?? page.params?.date,
+                })),
         ),
     );
 
     if (site.feed) {
-        const section = text(site.feed.section);
-        const time = (page) => new Date(text(page.date ?? page.params?.date)).getTime() || 0;
-        const posts = tree
-            .filter((page) => page.kind === "page" && text(page.section) === section)
-            .sort((a, b) => time(b) - time(a));
         write(
             path.join(out, ...section.split("/").filter(Boolean), "atom.xml"),
             atomFeed({
@@ -394,7 +469,7 @@ export function writeSite({ site, pages, out, report, staticDir = SITE_STATIC_DI
                 self: origin(feedUrl),
                 author: site.params?.author,
                 entries: posts.map((post) => {
-                    const words = plainText(post.content).split(" ").filter(Boolean);
+                    const words = plainText(contentOf(post)).split(" ").filter(Boolean);
                     return {
                         title: post.title,
                         url: origin(post.url),
@@ -405,7 +480,7 @@ export function writeSite({ site, pages, out, report, staticDir = SITE_STATIC_DI
                             (words.length > SUMMARY_WORDS ?
                                 `${words.slice(0, SUMMARY_WORDS).join(" ")}…`
                             :   words.join(" ")),
-                        content: post.content,
+                        content: contentOf(post),
                     };
                 }),
             }),
@@ -420,7 +495,7 @@ export function writeSite({ site, pages, out, report, staticDir = SITE_STATIC_DI
         write(path.join(out, "robots.txt"), robotsTxt([index, ...all]));
     }
 
-    return { files, pages: tree.map((page) => page.url), findings: theme.findings };
+    return result();
 }
 
 /**