@heroiclands/package-build 24.1.0 → 25.0.0

package/engine/content-scripts.mjs

@@ -92,7 +92,7 @@
 import MarkdownIt from "markdown-it";
 
 import { scanBlocks, BLOCK_CONTAINERS } from "./content-blocks.mjs";
-import { scanVerse, hasColumns, verseLine, verseGroups } from "./content-verse.mjs";
+import { scanVerse, hasColumns, verseLine, verseGroups, STANZA_BREAK } from "./content-verse.mjs";
 import { scanSpans } from "./content-spans.mjs";
 import { scanScrolls, scrollParagraphs } from "./content-scroll.mjs";
 import { directionsFromRecords } from "./script-direction.mjs";
@@ -113,7 +113,9 @@ import { unknownWordMessage, unknownWords, wordBooksFromRecords } from "./run-wo
 import { metreBooksFromRecords, metreFindings, metresFromRecords } from "./verse-metre.mjs";
 import { applyMap, mapsFromRecords } from "./text-maps.mjs";
 import { writtenInOf } from "./written-in.mjs";
-import { WIKILINK, parseWikilink } from "./wikilink-syntax.mjs";
+import { romanInventoriesOf, romanTextProblems } from "./roman-characters.mjs";
+import { isSettingScriptCodePoint } from "./content-charset.mjs";
+import { WIKILINK } from "./wikilink-syntax.mjs";
 import { matchAllOutsideCode } from "./code-fences.mjs";
 import { parseExtensionAttributes } from "./extension-attributes.mjs";
 
@@ -159,6 +161,8 @@ const CONFIG = "package-build.config.yaml";
  * @property {ReadonlyMap<string, import("./verse-metre.mjs").MetreBook>} [metreBooks] -
  *   What each configured language's verse is counted by, from
  *   {@link module:engine/verse-metre.metreBooksFromRecords}.
+ * @property {ReadonlyMap<string, import("./roman-characters.mjs").RomanInventory>} [romanInventories] -
+ *   Exact Roman source inventories by configured alias and BCP 47 tag.
  * @property {ReadonlyMap<string, import("./script-emphasis.mjs").Emphasis>} [emphases] -
  *   Each face's emphasis conventions, from the script notes, by
  *   {@link module:engine/script-emphasis.emphasesFromRecords}.
@@ -241,6 +245,7 @@ export function runContext(config, records) {
         ...base,
         maps,
         writtenIn: writtenInOf(languageRecords, maps),
+        romanInventories: romanInventoriesOf(languageRecords, config.languages),
         lexicons: lexiconsFromRecords(records, config),
         wordBooks: wordBooksFromRecords(records, config),
         metres: metresFromRecords(records),
@@ -1176,12 +1181,11 @@ function analyseRuns(source, ctx, syntax) {
             );
     };
 
+    const rawReferences = [];
     // Raw wikilinks belong to the resolver, not the span rewriter. Validate
     // their own control attributes with the same grammar before resolution,
     // without mapping a destination or consuming the trailing attributes.
     for (const match of matchAllOutsideCode(text, new RegExp(WIKILINK.source, "g"))) {
-        const parsed = parseWikilink(match[1], match[0]);
-        if (!parsed.attributes) continue;
         const close = match[0].indexOf("]]") + 1;
         const from = match.index + close + 2;
         const attrs = parseExtensionAttributes(match[0].slice(close + 2, -1), { positions: true });
@@ -1200,6 +1204,19 @@ function analyseRuns(source, ctx, syntax) {
         };
         const unchecked = errors.length;
         checkOwn(construct);
+        const bar = match[1].indexOf("|");
+        if (bar >= 0 && errors.length === unchecked) {
+            const ancestors = around(construct);
+            const inherited = ancestors.filter((parent) => parent.keys.lang !== undefined).at(-1)
+                ?.keys.lang;
+            rawReferences.push({
+                lang: construct.keys.lang ?? inherited,
+                from: match.index + 2 + bar + 1,
+                to: match.index + close - 1,
+                where: construct.where,
+                construct,
+            });
+        }
         if (errors.length !== unchecked || !native(construct)) continue;
         const keys = { ...construct.keys };
         nativeDefaults(construct, keys);
@@ -1215,6 +1232,9 @@ function analyseRuns(source, ctx, syntax) {
 
     const verseMarkers = new Map();
     for (const poem of scanVerse(text).blocks) {
+        for (let line = poem.start + 1; line < poem.end; line++)
+            if (STANZA_BREAK.test(lines[line].trim()))
+                verseMarkers.set(starts[line], starts[line] + lines[line].length);
         if (hasColumns(poem)) {
             for (const group of verseGroups(poem, lines)) {
                 if (group.stanzaBreak) continue;
@@ -1401,6 +1421,47 @@ function analyseRuns(source, ctx, syntax) {
         }
     };
 
+    /** The configured source inventory for either an alias or its BCP 47 tag. */
+    const inventoryOf = (lang) =>
+        lang === undefined ? undefined : (
+            (ctx.romanInventories?.get(lang) ?? ctx.romanInventories?.get(lang.toLowerCase()))
+        );
+
+    /** Check visible source before mapping or lexicon-native replacement. */
+    const roman = (parts, detail) => {
+        const inventory = inventoryOf(detail.run.lang);
+        if (!inventory) return;
+        if (!inventory.characters) {
+            report(
+                { line: detail.run.line, column: detail.run.column },
+                `the language note for \`${inventory.language}\` must declare \`data.roman\``,
+            );
+            return;
+        }
+        for (const [from, to] of parts) {
+            let base = from;
+            for (const line of text.slice(from, to).split("\n")) {
+                // A terminal unescaped backslash is a Markdown line break.
+                const breakMark =
+                    /(?<!\\)(?:\\\\)*\\$/.test(line) && text[base + line.length] === "\n";
+                const plain = unescapeRoman(breakMark ? line.slice(0, -1) : line);
+                for (const problem of romanTextProblems(plain.text, inventory, (char) => {
+                    const point = char.codePointAt(0);
+                    return (
+                        detail.run.nativefont === true &&
+                        isSettingScriptCodePoint(point) &&
+                        detail.face?.draws.has(point)
+                    );
+                }))
+                    report(
+                        positionOf(starts, base + (plain.offsets[problem.offset] ?? 0)),
+                        problem.message,
+                    );
+                base += line.length + 1;
+            }
+        }
+    };
+
     /**
      * Report each word of a mapped run that is not a word of its language, once
      * per run, at its first character.
@@ -1515,6 +1576,7 @@ function analyseRuns(source, ctx, syntax) {
         };
         runs.push(run);
         const plain =
+            inventoryOf(lang) !== undefined ||
             font !== undefined ||
             map !== undefined ||
             (lang !== undefined &&
@@ -1577,15 +1639,18 @@ function analyseRuns(source, ctx, syntax) {
             }
             if (plain && errors.length === before)
                 plainInline(inline, from, marking, detail, detail.holes);
-            if (named && errors.length === before) {
+            if (errors.length === before) {
                 const parts = outsideDelimiters(
                     outsideHoles(from, to, detail.holes),
                     detail.emphasis,
                 );
-                lexical(parts, detail);
-                capitals(parts, detail);
-                writings(parts, detail);
-                covered(parts, detail);
+                roman(parts, detail);
+                if (named) {
+                    lexical(parts, detail);
+                    capitals(parts, detail);
+                    writings(parts, detail);
+                    covered(parts, detail);
+                }
             }
             continue;
         }
@@ -1647,15 +1712,30 @@ function analyseRuns(source, ctx, syntax) {
             for (let line = first; line < last; line++)
                 plainInline(lines[line], starts[line], marking, detail, detail.holes);
         }
-        if (named && errors.length === before) {
+        if (errors.length === before) {
             const parts = outsideDelimiters(outsideHoles(from, to, detail.holes), detail.emphasis);
-            lexical(parts, detail);
-            capitals(parts, detail);
-            writings(parts, detail);
-            covered(parts, detail);
+            roman(parts, detail);
+            if (named) {
+                lexical(parts, detail);
+                capitals(parts, detail);
+                writings(parts, detail);
+                covered(parts, detail);
+            }
         }
     }
 
+    for (const reference of rawReferences) {
+        const font = reference.construct.keys.font ?? ctx.writtenIn?.get(reference.lang)?.font;
+        roman([[reference.from, reference.to]], {
+            run: {
+                lang: reference.lang,
+                nativefont: native(reference.construct),
+                ...reference.where,
+            },
+            face: font === undefined ? undefined : faces.get(font),
+        });
+    }
+
     const order = (a, b) => a.line - b.line || a.column - b.column;
     runs.sort(order);
     details.sort((a, b) => order(a.run, b.run));

package/engine/content-session.mjs

@@ -13,6 +13,9 @@
 
 /** One invocation's authored sources and context-specific prepared views. */
 import path from "node:path";
+import fs from "node:fs";
+import { DEPLOYED_ASSET_TYPES } from "./asset-types.mjs";
+import { createContentTimings } from "./content-timings.mjs";
 import * as contentIndex from "./content-index.mjs";
 import * as blocks from "./content-blocks.mjs";
 import * as alerts from "./content-alerts.mjs";
@@ -52,6 +55,8 @@ import { lintNoteStates } from "./stub-lint.mjs";
  * @property {() => Promise<import("./content-index.mjs").PreparedContentIndex>} publication
  * @property {() => ReturnType<import("./compile-corpus.mjs").buildCompileCorpus>} compileCorpus
  * @property {(targets: readonly ContentTarget[]) => Promise<ContentSession>} prepare
+ * @property {ReturnType<typeof createContentTimings>} timings
+ * @property {() => void} assertUnchanged
  * @property {() => void} dispose
  */
 
@@ -71,6 +76,8 @@ export const CONTENT_TARGETS = Object.freeze(["index", "lint", "links", "compile
  * @param {string} [options.assetsBase] - Assets root, defaulting to configuration.
  * @param {readonly string[]} [options.skipDirectories] - Resolved content scope.
  * @param {ReturnType<typeof routerFor>} [options.router] - One router shared by this session's compile.
+ * @param {boolean} [options.trackChanges] - Snapshot physical input metadata for build phase guards.
+ * @param {ReturnType<typeof createContentTimings>} [options.timings] - Invocation stage accounting.
  * @returns {Promise<ContentSession>} Captured sources and lazy prepared views; dispose after use.
  */
 export async function createContentSession({
@@ -79,14 +86,36 @@ export async function createContentSession({
     assetsBase = config.paths.assets,
     skipDirectories = config.skipDirectories,
     router = routerFor(config),
+    timings = createContentTimings(),
+    trackChanges = false,
 } = {}) {
     const tree = path.resolve(contentBase);
     const assets = path.resolve(assetsBase);
     const scope = [...skipDirectories];
-    const inventory = captureContentSources(tree, assets, scope);
+    const inventory = timings.sync("source capture", () =>
+        captureContentSources(tree, assets, scope),
+    );
+    const fingerprint = (file) => {
+        const stat = fs.statSync(file, { throwIfNoEntry: false });
+        return stat ? `${stat.ino}:${stat.size}:${stat.mtimeMs}:${stat.ctimeMs}` : "absent";
+    };
+    const fingerprints =
+        trackChanges ?
+            new Map(
+                [
+                    ...new Set([
+                        tree,
+                        ...DEPLOYED_ASSET_TYPES.map((entry) => path.resolve(assets, entry.root)),
+                        ...inventory.paths.map((entry) => entry.file),
+                    ]),
+                ].map((file) => [file, fingerprint(file)]),
+            )
+        :   undefined;
     const parsed = new Map();
     const problems = [];
-    const foreign = loadForeignIndexes(config, [config.contentPackage]);
+    const foreign = timings.sync("dependency indexes", () =>
+        loadForeignIndexes(config, [config.contentPackage]),
+    );
     const syntaxCache = new Map();
     const prepared = new Map();
     let disposed = false;
@@ -104,23 +133,31 @@ export async function createContentSession({
             });
         return syntaxCache.get(text);
     };
-    const records = contentIndex.indexRecordsFor({
-        config,
-        contentBase: tree,
-        assetsBase: assets,
-        skipDirectories: scope,
-        sources: inventory.indexSources,
-        parsed,
-        problems,
-        foreignIndex: foreign.index,
-        syntax,
-    });
+    const records = timings.sync("index derivation", () =>
+        contentIndex.indexRecordsFor({
+            config,
+            contentBase: tree,
+            assetsBase: assets,
+            skipDirectories: scope,
+            sources: inventory.indexSources,
+            parsed,
+            problems,
+            foreignIndex: foreign.index,
+            syntax,
+        }),
+    );
     const memo = (key, make) => {
         active();
-        if (!prepared.has(key)) prepared.set(key, Promise.resolve().then(make));
+        if (!prepared.has(key))
+            prepared.set(
+                key,
+                Promise.resolve().then(() => timings.measure(key, make)),
+            );
+        else timings.hit(key);
         return prepared.get(key);
     };
     const session = {
+        timings,
         config,
         contentBase: tree,
         assetsBase: assets,
@@ -133,6 +170,17 @@ export async function createContentSession({
         problems,
         foreign,
         syntax,
+        /** Refuse a stale capture after any build phase changes an input. */
+        assertUnchanged() {
+            active();
+            if (!fingerprints)
+                throw new Error("Content change tracking was not enabled for this session");
+            for (const [file, before] of fingerprints)
+                if (fingerprint(file) !== before)
+                    throw new Error(
+                        `Content sources changed during the build: ${file}. Finish source generation before content build.`,
+                    );
+        },
         /** Refuse use with a different configuration, root, scope or router. */
         assertContext(options = {}) {
             active();

package/engine/content-timings.mjs

@@ -0,0 +1,67 @@
+/*
+ * This file is part of the Song of Heroic Lands (SoHL) system for Foundry VTT.
+ * Copyright (c) 2024-2026 Tom Rodriguez ("Toasty") — <toasty@heroiclands.org>
+ *
+ * This work is licensed under the GNU General Public License v3.0 (GPLv3).
+ * You may copy, modify, and distribute it under the terms of that license.
+ *
+ * For full terms, see the LICENSE.md file in the project root or visit:
+ * https://www.gnu.org/licenses/gpl-3.0.html
+ *
+ * SPDX-License-Identifier: GPL-3.0-or-later
+ */
+
+/** Sequential, nested stage accounting for one content invocation. */
+export function createContentTimings() {
+    const started = performance.now();
+    let changed = started;
+    const stack = ["orchestration"];
+    const stages = new Map();
+    const hits = new Map();
+    const account = () => {
+        const now = performance.now();
+        const name = stack.at(-1);
+        stages.set(name, (stages.get(name) ?? 0) + now - changed);
+        changed = now;
+    };
+    const begin = (name) => {
+        account();
+        stack.push(name);
+        return () => {
+            account();
+            stack.pop();
+        };
+    };
+    return {
+        /** @template T @param {string} name @param {() => T} work @returns {T} */
+        sync(name, work) {
+            const stop = begin(name);
+            try {
+                return work();
+            } finally {
+                stop();
+            }
+        },
+        /** @template T @param {string} name @param {() => Promise<T>|T} work @returns {Promise<T>} */
+        async measure(name, work) {
+            const stop = begin(name);
+            try {
+                return await work();
+            } finally {
+                stop();
+            }
+        },
+        /** @param {string} name */
+        hit(name) {
+            hits.set(name, (hits.get(name) ?? 0) + 1);
+        },
+        snapshot() {
+            account();
+            return {
+                totalMs: changed - started,
+                stages: Object.fromEntries(stages),
+                hits: Object.fromEntries(hits),
+            };
+        },
+    };
+}

package/engine/index.mjs

@@ -227,6 +227,9 @@ export * as contentLint from "./content-lint.mjs";
 
 /** The charset authored content is held to, so a book can choose its face. */
 export * as contentCharset from "./content-charset.mjs";
+
+/** Exact permitted Roman source characters for configured languages. */
+export * as romanCharacters from "./roman-characters.mjs";
 export * as pathCharset from "./path-charset.mjs";
 
 /** Naming an interface icon in a note, without drawing one there. */

package/engine/language-data.mjs

@@ -73,6 +73,7 @@ import {
 import { LANGUAGE_SUBTYPE } from "./word-terms.mjs";
 import { WORD_CLASS_BASES } from "./word-classes.mjs";
 import { nearest } from "./near-miss.mjs";
+import { romanInventoryProblems } from "./roman-characters.mjs";
 
 /** The bases a frame's `class` names. */
 const CLASS_BASES = new Set(WORD_CLASS_BASES.map((item) => item.token));
@@ -614,7 +615,9 @@ export function checkLanguageData(note, { index, types } = {}) {
             ),
         );
 
-    const findings = [];
+    const findings = romanInventoryProblems(data.roman).map((message) =>
+        finding(["data", "roman"], "error", message),
+    );
 
     // Only the shape of a supplied declaration: no word is compared with it.
     const grammar = isMap(data.grammar) ? data.grammar : {};

package/engine/language-terms.mjs

@@ -30,6 +30,7 @@
 
 /** The `data` keys a language note states its sounds, names and history in. */
 export const LANGUAGE_DATA_KEYS = Object.freeze([
+    "roman",
     "tongues",
     "register",
     "retired",
@@ -1889,6 +1890,15 @@ const GRAMMAR_FIELDS = Object.freeze([
  * @type {readonly object[]}
  */
 export const LANGUAGE_DATA_FIELDS = Object.freeze([
+    Object.freeze({
+        name: "roman",
+        kind: "string",
+        shape: "a literal NFC string of permitted characters, including case, accents, digits and punctuation",
+        describe:
+            "The exact Roman source characters this language permits, each once. No ranges, case " +
+            "expansion or inherited inventory; spaces and line breaks are structural. An empty " +
+            "string declares no written characters. Required on each configured language note.",
+    }),
     Object.freeze({
         name: "grammar",
         kind: "map",

package/engine/map-lexicon.mjs

@@ -330,7 +330,18 @@ export function languageNoteFindings(config, records) {
     }
     const findings = [];
     for (const [name, { record }] of languageNotes(languages, contentPackage, records)) {
-        if (record) continue;
+        if (record) {
+            if (typeof record.data?.roman !== "string")
+                findings.push({
+                    file: configFile ?? CONFIG,
+                    ...(text ?
+                        positionOfYamlPath(text, ["languages", name, "note"], { key: true })
+                    :   {}),
+                    severity: "error",
+                    message: `the language note named by \`languages.${name}.note\` must declare \`data.roman\``,
+                });
+            continue;
+        }
         findings.push({
             file: configFile ?? CONFIG,
             ...(text ? positionOfYamlPath(text, ["languages", name, "note"], { key: true }) : {}),

package/engine/roman-characters.mjs

@@ -0,0 +1,113 @@
+/*
+ * This file is part of the Song of Heroic Lands (SoHL) system for Foundry VTT.
+ * Copyright (c) 2024-2026 Tom Rodriguez ("Toasty") — <toasty@heroiclands.org>
+ *
+ * This work is licensed under the GNU General Public License v3.0 (GPLv3).
+ * You may copy, modify, and distribute it under the terms of that license.
+ *
+ * For full terms, see the LICENSE.md file in the project root or visit:
+ * https://www.gnu.org/licenses/gpl-3.0.html
+ *
+ * SPDX-License-Identifier: GPL-3.0-or-later
+ */
+
+/** A language's exact Roman source characters, before any script conversion. */
+
+/** Unicode graphemes keep every canonically composing sequence together. */
+const graphemes = new Intl.Segmenter(undefined, { granularity: "grapheme" });
+
+/** A code point as the author sees it in a finding. */
+const codeName = (char) => `U+${char.codePointAt(0).toString(16).toUpperCase().padStart(4, "0")}`;
+
+/**
+ * Semantic problems in a supplied inventory. Shape belongs to the vocabulary.
+ * An empty string explicitly declares no written characters.
+ * @param {unknown} value - The authored `data.roman`.
+ * @returns {string[]} Findings at that field.
+ */
+export function romanInventoryProblems(value) {
+    if (typeof value !== "string") return [];
+    const problems = [];
+    if (value !== value.normalize("NFC"))
+        problems.push(
+            "`data.roman` must use NFC spelling; write precomposed accented letters where available",
+        );
+    if (/\s/u.test(value))
+        problems.push(
+            "`data.roman` lists characters without whitespace; spaces and line breaks are structural",
+        );
+    if (/[\p{Cc}\p{Cf}\p{Cs}]/u.test(value))
+        problems.push("`data.roman` contains a control, format or unpaired surrogate character");
+    const seen = new Set();
+    const repeated = new Set();
+    for (const char of value) {
+        if (seen.has(char)) repeated.add(char);
+        seen.add(char);
+    }
+    if (repeated.size)
+        problems.push(
+            `\`data.roman\` repeats ${[...repeated].map((char) => `\`${char}\` (${codeName(char)})`).join(", ")}; list each character once`,
+        );
+    return problems;
+}
+
+/**
+ * @typedef {object} RomanInventory
+ * @property {string} language - The configured language name.
+ * @property {ReadonlySet<string>|undefined} characters - Exact code points, absent for a missing declaration.
+ */
+
+/**
+ * Each configured language note's own inventory, by alias and BCP 47 tag.
+ * Language-family ancestry does not supply an inventory. Unassociated external
+ * tags have no entry and keep their ordinary behavior.
+ * @param {ReadonlyMap<string, Record<string, any>>} records - Configured language notes.
+ * @param {Readonly<Record<string, {tag: string}>>} languages - Language configuration.
+ * @returns {Map<string, RomanInventory>}
+ */
+export function romanInventoriesOf(records, languages = {}) {
+    const out = new Map();
+    for (const [language, record] of records) {
+        const value = record.data?.roman;
+        const inventory = Object.freeze({
+            language,
+            characters: typeof value === "string" ? new Set(value) : undefined,
+        });
+        out.set(language, inventory);
+        const tag = languages[language]?.tag;
+        if (typeof tag === "string") out.set(tag.toLowerCase(), inventory);
+    }
+    return out;
+}
+
+/**
+ * Invalid source characters, retaining offsets in the supplied unescaped text.
+ * No punctuation, case or normalization repair is performed.
+ * @param {string} text - Visible Roman source.
+ * @param {RomanInventory} inventory - Its language's declared characters.
+ * @param {(char: string) => boolean} [native] - Whether a directly typed native sign is drawn in the selected native face.
+ * @returns {Array<{offset: number, message: string}>}
+ */
+export function romanTextProblems(text, inventory, native = () => false) {
+    const out = [];
+    if (!inventory.characters) return out;
+    for (const { segment: cluster, index } of graphemes.segment(text)) {
+        let offset = index;
+        if (cluster !== cluster.normalize("NFC")) {
+            out.push({
+                offset,
+                message: `\`${cluster}\` is not NFC spelling in \`${inventory.language}\`; write \`${cluster.normalize("NFC")}\``,
+            });
+            continue;
+        }
+        for (const char of cluster) {
+            if (!/[ \t\r\n]/u.test(char) && !inventory.characters.has(char) && !native(char))
+                out.push({
+                    offset,
+                    message: `\`${char}\` (${codeName(char)}) is not in \`${inventory.language}\`'s \`data.roman\` inventory`,
+                });
+            offset += char.length;
+        }
+    }
+    return out;
+}