@heroiclands/package-build 25.1.1 → 25.1.2

package/CHANGELOG.md

@@ -1,5 +1,12 @@
 # @heroiclands/package-build
 
+## 25.1.2
+
+**Build speed**
+
+- Content builds and content lint run several times faster on large packages, above all when building the content index and checking text written through a language's encoding.
+- The build log shows how long each stage of the content index takes.
+
 ## 25.1.1
 
 **Language conversion**

package/bin/content-commands.mjs

@@ -1912,6 +1912,40 @@ function linksCommand() {
     };
 }
 
+/** The stages that build the content index, in the order they run. */
+const INDEX_STAGES = Object.freeze([
+    "source capture",
+    "dependency indexes",
+    "index read",
+    "index notes",
+    "index citations",
+    "index encoding runs",
+    "index calendars and cultures",
+    "index lexical sections",
+    "index order",
+    "index derivation",
+    "sql:all",
+    "sql:public",
+    "publication",
+    "index emission",
+]);
+
+/**
+ * The content index's stage timings as one line, each stage's own time
+ * excluding the stages inside it.
+ *
+ * @param {Record<string, number>} stages - Exclusive milliseconds by stage.
+ * @returns {string} The line.
+ */
+function indexTimings(stages) {
+    const shown = INDEX_STAGES.filter((name) => name in stages);
+    const total = shown.reduce((sum, name) => sum + stages[name], 0);
+    return (
+        `Index timings: ${shown.map((name) => `${name} ${stages[name].toFixed(1)} ms`).join("; ")}; ` +
+        `total ${total.toFixed(1)} ms.`
+    );
+}
+
 /**
  * `package-build content-index` — emit this package's note index.
  *
@@ -1961,12 +1995,14 @@ function contentIndexCommand() {
                 session.assertContext({ config, contentBase });
                 const publication = await session.publication();
                 const { file, notes, bytes, full, draft, stub, dictionary, terms } =
-                    emitContentIndex({
-                        config,
-                        publication,
-                        ...(argv.root ? { contentBase: argv.root } : {}),
-                        ...(argv.out ? { outDir: argv.out } : {}),
-                    });
+                    session.timings.sync("index emission", () =>
+                        emitContentIndex({
+                            config,
+                            publication,
+                            ...(argv.root ? { contentBase: argv.root } : {}),
+                            ...(argv.out ? { outDir: argv.out } : {}),
+                        }),
+                    );
                 log.info(
                     `${config.contentPackage} → ${path.relative(process.cwd(), file)} ` +
                         `(${notes} notes — ${full} full, ${draft} draft, ` +
@@ -1977,6 +2013,7 @@ function contentIndexCommand() {
                         `${config.contentPackage} → ${path.relative(process.cwd(), written.file)} ` +
                             `(${written.rows} ${relation} rows)`,
                     );
+                log.info(indexTimings(session.timings.snapshot().stages));
                 return true;
             } catch (err) {
                 reportContentFailure(err, argv);

package/engine/content-alerts.mjs

@@ -12,6 +12,7 @@
  */
 
 import MarkdownIt from "markdown-it";
+import { blockTokens } from "./markdown-blocks.mjs";
 import { parseExtensionAttributes, refusedAttributes } from "./extension-attributes.mjs";
 import { slugify } from "./content-slug.mjs";
 
@@ -48,7 +49,6 @@ export const ALERT_TYPES = Object.freeze({
         ),
     },
 });
-const parser = new MarkdownIt({ html: true });
 const titleParser = new MarkdownIt({ html: false });
 const marker = /^ {0,3}>[ \t]?\[!([^\]\n]+)\](.*)$/;
 /** Read alerts outside code examples, retaining inclusive source line ranges.
@@ -60,7 +60,7 @@ export function scanAlerts(source) {
         blocks = [],
         errors = [];
     let depth = 0;
-    for (const token of parser.parse(text, {})) {
+    for (const token of blockTokens(text)) {
         if (token.type === "blockquote_close") {
             depth--;
             continue;

package/engine/content-blocks.mjs

@@ -52,6 +52,7 @@
 import crypto from "node:crypto";
 import { slugify } from "./content-slug.mjs";
 import MarkdownIt from "markdown-it";
+import { blockTokens } from "./markdown-blocks.mjs";
 import { parseExtensionAttributes, refusedAttributes } from "./extension-attributes.mjs";
 import { scanVerse, renderVerse } from "./content-verse.mjs";
 import { renderScroll, scanScrolls } from "./content-scroll.mjs";
@@ -63,7 +64,6 @@ import { WITHHELD_CLASS, parseHeadingLine, withheldSections } from "./heading-at
  * and a stray `<section>` would close the block early.
  */
 const titleParser = new MarkdownIt({ html: false });
-const structureParser = new MarkdownIt({ html: true });
 
 /** The blocks an author may open, with the heading each takes by default. */
 export const BLOCK_NAMES = Object.freeze({
@@ -108,7 +108,7 @@ export function scanBlocks(source) {
     // A bare fence can belong to a quote through lazy continuation.
     // Parser ranges decide whether it can delimit a container.
     const literalLines = new Set();
-    for (const token of structureParser.parse(String(source ?? ""), {})) {
+    for (const token of blockTokens(source)) {
         if (
             !token.map ||
             !["blockquote_open", "fence", "code_block", "html_block"].includes(token.type)

package/engine/content-index.mjs

@@ -1087,6 +1087,8 @@ export function resolveCultures(
  * @param {Map<string, {raw: string}>} [options.sources] - Selected captured source strings.
  * @param {import("./content-session.mjs").ContentSyntax} [options.syntax] - Session scanner projections.
  * @param {Iterable<string>} [options.nameTypes] - Additional bare name kinds from configured type declarations.
+ * @param {ReturnType<typeof import("./content-timings.mjs").createContentTimings>} [options.timings] -
+ *   Invocation stage accounting, which records each phase of the derivation.
  * @returns {Array<Record<string, any>>} The records, in a total order that does
  *   not depend on directory-read order.
  */
@@ -1109,8 +1111,16 @@ export function collectContentIndex(
         sources,
         syntax,
         nameTypes,
+        timings,
     },
 ) {
+    // Each phase runs until the next begins; the caller's stage closes the last.
+    let phaseEnd;
+    const phase = (name) => {
+        phaseEnd?.();
+        phaseEnd = timings?.start(name);
+    };
+    phase("index read");
     const records = [];
     // Passed through rather than defaulted away: an absent scope is the
     // caller's omission, and `walkMarkdownTree` says so.
@@ -1175,6 +1185,7 @@ export function collectContentIndex(
     );
 
     // Each recorded note with its body, for the words its prose cites.
+    phase("index notes");
     const bodies = [];
     for (const { frontmatter, body, bodyLine, bodyColumn, raw, absPath } of parsedNotes) {
         // Before the index derives anything onto the frontmatter, so a later
@@ -1251,16 +1262,19 @@ export function collectContentIndex(
 
     // The words each note's prose cites, read once every language's words
     // are recorded; the index keeps the citing Addresses and no text.
+    phase("index citations");
     attachCitations(records, bodies);
 
     // A run marked with a language alone takes its language's script, which
     // is read once every language note is recorded.
+    phase("index encoding runs");
     fillEncodingRuns(records, { languages, contentPackage, rootDir });
 
     // Every note record exists and every Address in it is a tuple, so a parent
     // is found by its canonical address. Before the sort, which reads nothing
     // the pass writes. Calendars first: they read the culture a note states,
     // which the culture pass then replaces with the one it resolves.
+    phase("index calendars and cultures");
     resolveNoteCalendars(records, {
         contentPackage,
         defaultCalendar,
@@ -1298,6 +1312,7 @@ export function collectContentIndex(
     // projection as the surfaces, without making the index emitter async.
     // Public notes resolve providers in their audience, including lineage
     // through hidden languages, just as their build does.
+    phase("index lexical sections");
     const visible = records.filter((record) => !isGmNote(record));
     const publicSections = lexicalSectionsFor(visible, records);
     const allSections = lexicalSectionsFor(records);
@@ -1319,6 +1334,7 @@ export function collectContentIndex(
     // about the content, and the artifact is meant to be byte-identical across
     // two runs over an unchanged tree. Collected above, ahead of the notes —
     // see `resolveRole` — rather than walked a second time here.
+    phase("index order");
     for (const record of assetRecords) {
         records.push(/** @type {Record<string, any>} */ (sortKeysDeep(record)));
     }
@@ -1344,9 +1360,11 @@ export function collectContentIndex(
             ) ||
             String(a.id ?? "").localeCompare(String(b.id ?? ""), "en"),
     );
-    return records.map((record) =>
+    const decoded = records.map((record) =>
         decodeIndexAddresses(record, addressContext ?? { package: contentPackage }),
     );
+    phaseEnd?.();
+    return decoded;
 }
 
 /**
@@ -1465,6 +1483,8 @@ function dependencyAddresses(config) {
  * @param {import("./content-session.mjs").ContentSyntax} [opts.syntax] - Session scanner projections.
  * @param {Iterable<string>} [opts.nameTypes] - Bare name kinds, defaulting to configured item types.
  * @param {Map<string, object>} [opts.foreignIndex] - Captured dependency address view.
+ * @param {ReturnType<typeof import("./content-timings.mjs").createContentTimings>} [opts.timings] -
+ *   Invocation stage accounting.
  * @returns {object[]} One record per note, plus one per documentation entry.
  */
 export function indexRecordsFor({
@@ -1478,6 +1498,7 @@ export function indexRecordsFor({
     syntax,
     nameTypes,
     foreignIndex,
+    timings,
 } = {}) {
     const resolved = config ?? loadPackConfig();
     const tree = contentBase ?? resolved.paths.content;
@@ -1512,6 +1533,7 @@ export function indexRecordsFor({
         sources,
         syntax,
         nameTypes: nameTypes ?? resolved.itemTypes,
+        timings,
     });
 }
 

package/engine/content-scripts.mjs

@@ -41,6 +41,7 @@
  */
 
 import MarkdownIt from "markdown-it";
+import { blockTokens } from "./markdown-blocks.mjs";
 
 import { scanBlocks, BLOCK_CONTAINERS } from "./content-blocks.mjs";
 import { scanVerse, hasColumns, verseLine, verseGroups, STANZA_BREAK } from "./content-verse.mjs";
@@ -737,9 +738,6 @@ function codeName(point) {
     return `U+${point.toString(16).toUpperCase().padStart(4, "0")}`;
 }
 
-/** A parser for the block structure of a div's body. */
-const blockParser = new MarkdownIt({ html: true });
-
 /** What a block token is called in a finding. */
 const BLOCK_WHAT = Object.freeze({
     heading_open: "a heading",
@@ -1485,7 +1483,7 @@ function analyseRuns(source, ctx, syntax) {
                     { line: first + line + 1, column: /^ */.exec(body[line])[0].length + 1 },
                     `a div marked \`${marking}\` holds paragraphs and line breaks only, not ${what}`,
                 );
-            for (const token of blockParser.parse(body.join("\n"), {})) {
+            for (const token of blockTokens(body.join("\n"))) {
                 if (token.level !== 0 || !token.map) continue;
                 const [top, past] = token.map;
                 if (token.type === "paragraph_open") {
@@ -1535,7 +1533,14 @@ function analyseRuns(source, ctx, syntax) {
     const boundsOf = (detail) =>
         detail.reference ??
         (detail.run.kind === "span" ? [detail.run.start + 1, detail.contentEnd] : detail.body);
+    // Each run's parent and owner are read many times while ownership is
+    // assigned below, so each is found once.
+    const parents = new Map();
     const parentOf = (detail) => {
+        if (!parents.has(detail)) parents.set(detail, findParent(detail));
+        return parents.get(detail);
+    };
+    const findParent = (detail) => {
         if (detail.run.kind !== "span" || Object.keys(detail.keys).length) return undefined;
         return details
             .filter(
@@ -1547,9 +1552,13 @@ function analyseRuns(source, ctx, syntax) {
             )
             .sort((a, b) => boundsOf(a)[1] - boundsOf(a)[0] - (boundsOf(b)[1] - boundsOf(b)[0]))[0];
     };
+    const owners = new Map();
     const ownerOf = (detail) => {
-        const parent = parentOf(detail);
-        return parent ? ownerOf(parent) : detail;
+        if (!owners.has(detail)) {
+            const parent = parentOf(detail);
+            owners.set(detail, parent ? ownerOf(parent) : detail);
+        }
+        return owners.get(detail);
     };
     const wholeVerseReading = (detail) => {
         if (detail.run.kind === "verse") return true;

package/engine/content-scroll.mjs

@@ -42,7 +42,7 @@
  * @module
  */
 
-import MarkdownIt from "markdown-it";
+import { blockTokens } from "./markdown-blocks.mjs";
 
 import { parseExtensionAttributes, refusedAttributes } from "./extension-attributes.mjs";
 
@@ -78,8 +78,6 @@ export const BOOK_SCROLL_ROWS = 40;
 /** The class every scroll carries. */
 export const SCROLL_CLASS = "scroll";
 
-const structureParser = new MarkdownIt({ html: true });
-
 /**
  * One `scroll` fence.
  *
@@ -129,7 +127,7 @@ export function scanScrolls(source) {
     if (!/^ *(?:`{3,}|~{3,}) *scroll(?:\s|$)/m.test(text)) return { blocks, errors };
     const lines = text.split("\n");
     const literalLines = new Set();
-    for (const token of structureParser.parse(text, {})) {
+    for (const token of blockTokens(text)) {
         if (!token.map || !["blockquote_open", "html_block"].includes(token.type)) continue;
         for (let line = token.map[0]; line < token.map[1]; line++) literalLines.add(line);
     }

package/engine/content-session.mjs

@@ -144,6 +144,7 @@ export async function createContentSession({
             problems,
             foreignIndex: foreign.index,
             syntax,
+            timings,
         }),
     );
     const memo = (key, make) => {

package/engine/content-timings.mjs

@@ -24,12 +24,15 @@ export function createContentTimings() {
         stages.set(name, (stages.get(name) ?? 0) + now - changed);
         changed = now;
     };
+    // Ending a stage also ends any stage opened inside it and left open.
     const begin = (name) => {
         account();
+        const depth = stack.length;
         stack.push(name);
         return () => {
+            if (stack.length <= depth) return;
             account();
-            stack.pop();
+            stack.length = depth;
         };
     };
     return {
@@ -51,6 +54,15 @@ export function createContentTimings() {
                 stop();
             }
         },
+        /**
+         * Open a stage that runs until the returned function is called, or
+         * until the stage it was opened in ends.
+         * @param {string} name
+         * @returns {() => void}
+         */
+        start(name) {
+            return begin(name);
+        },
         /** @param {string} name */
         hit(name) {
             hits.set(name, (hits.get(name) ?? 0) + 1);

package/engine/content-verse.mjs

@@ -180,11 +180,11 @@ function columnsMarkdown(block, lines, step) {
 }
 
 import MarkdownIt from "markdown-it";
+import { blockTokens } from "./markdown-blocks.mjs";
 import { slugify } from "./content-slug.mjs";
 import { spanMarkdownPlugin } from "./content-spans.mjs";
 import { parseExtensionAttributes, refusedAttributes } from "./extension-attributes.mjs";
 
-const structureParser = new MarkdownIt({ html: true });
 const lineParser = new MarkdownIt({ html: true }).use(spanMarkdownPlugin);
 
 /** The structural first span of a line, with its source ranges and keys. */
@@ -223,7 +223,7 @@ export function scanVerse(source) {
         blocks = [],
         errors = [];
     const literalLines = new Set();
-    for (const token of structureParser.parse(String(source ?? ""), {})) {
+    for (const token of blockTokens(source)) {
         if (!token.map || !["blockquote_open", "html_block"].includes(token.type)) continue;
         for (let line = token.map[0]; line < token.map[1]; line++) literalLines.add(line);
     }

package/engine/dictionary-rows.mjs

@@ -464,6 +464,16 @@ export function dictionaryAndTermRows(records, all = records) {
         );
         return hit ? hit.entry[key] : undefined;
     };
+    // One piece lookup per language, shared by every word that language states.
+    const glossers = new Map();
+    const glossesFor = (language) => {
+        if (!glossers.has(language))
+            glossers.set(
+                language,
+                pieceGlosses(...lineage(language).map((key) => languages.get(key).data)),
+            );
+        return glossers.get(language);
+    };
     const etymologyOf = ({ word, statedIn, depth }) => {
         const blank = Object.fromEntries(ETYMOLOGY_KEYS.map((key) => [key, null]));
         if (depth > 0)
@@ -476,13 +486,15 @@ export function dictionaryAndTermRows(records, all = records) {
             };
         const source = isMap(word.source) ? word.source : {};
         const built = Array.isArray(word.built) ? word.built.map(String) : [];
-        const glossOf = pieceGlosses(...lineage(statedIn).map((key) => languages.get(key).data));
         const out = {
             ...blank,
             kind: typeof source.kind === "string" ? source.kind : null,
             language: typeof source.language === "string" ? source.language : null,
             form: typeof source.form === "string" ? source.form : null,
-            parts: built.length ? built.map((form) => ({ form, gloss: glossOf(form) })) : null,
+            parts:
+                built.length ?
+                    built.map((form) => ({ form, gloss: glossesFor(statedIn)(form) }))
+                :   null,
             root: typeof word.root === "string" ? word.root : null,
             frame: typeof word.frame === "string" ? word.frame : null,
         };

package/engine/encoding-language.mjs

@@ -69,7 +69,14 @@ function capitalWriting(enc, word, native, font) {
             upper.to + native.slice(lower.to.length)
         :   undefined;
 }
+/** Each lexicon's forms, found once; a lexicon is not modified once it is converted with. */
+const formsOf = new WeakMap();
 function entryForms(lexicon) {
+    if (!lexicon) return findEntryForms(lexicon);
+    if (!formsOf.has(lexicon)) formsOf.set(lexicon, findEntryForms(lexicon));
+    return formsOf.get(lexicon);
+}
+function findEntryForms(lexicon) {
     const data = lexicon?.data ?? {};
     const derived = new Map();
     for (const entry of derivedForms(data)) {
@@ -122,6 +129,44 @@ function entryForms(lexicon) {
     ];
 }
 
+/** Each lexicon's phrases of more than one word, with the patterns that find them. */
+const phrasesFound = new WeakMap();
+function phrasesOf(lexicon) {
+    if (!phrasesFound.has(lexicon))
+        phrasesFound.set(
+            lexicon,
+            entryForms(lexicon)
+                .map((entry) => entry.form)
+                .filter((form) => [...form.matchAll(WORD)].length > 1)
+                .map((form) => ({
+                    form,
+                    spaces: form.match(/[\p{Zs}\u0009-\u000D]+/gu) ?? [],
+                    pattern: new RegExp(
+                        form
+                            .split(/[\p{Zs}\u0009-\u000D]+/u)
+                            .map((part) => part.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"))
+                            .join("[\\p{Zs}\\u0009-\\u000D]+"),
+                        "iuy",
+                    ),
+                })),
+        );
+    return phrasesFound.get(lexicon);
+}
+
+/** Each lexicon's words' forms and variants, as written. */
+const statedFound = new WeakMap();
+function statedForms(lexicon) {
+    if (!statedFound.has(lexicon)) {
+        const forms = new Set();
+        for (const entry of list(lexicon.data.words)) {
+            forms.add(entry.form);
+            for (const variant of list(entry.variants)) forms.add(variant);
+        }
+        statedFound.set(lexicon, forms);
+    }
+    return statedFound.get(lexicon);
+}
+
 function literalEnd(text, at) {
     let cursor = at + OPEN.length;
     while (cursor < text.length) {
@@ -164,23 +209,7 @@ export function encodeWithLanguage(enc, text, options = {}) {
     }
     const writing =
         options.lexicon ? writingsOf(options.lexicon.data, enc.name, options.font) : undefined;
-    const phrases =
-        options.lexicon ?
-            entryForms(options.lexicon)
-                .map((entry) => entry.form)
-                .filter((form) => [...form.matchAll(WORD)].length > 1)
-                .map((form) => ({
-                    form,
-                    spaces: form.match(/[\p{Zs}\u0009-\u000D]+/gu) ?? [],
-                    pattern: new RegExp(
-                        form
-                            .split(/[\p{Zs}\u0009-\u000D]+/u)
-                            .map((part) => part.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"))
-                            .join("[\\p{Zs}\\u0009-\\u000D]+"),
-                        "iuy",
-                    ),
-                }))
-        :   [];
+    const phrases = options.lexicon ? phrasesOf(options.lexicon) : [];
     let at = 0;
     const appendRows = (from, to) => {
         let cursor = from;
@@ -267,9 +296,7 @@ export function encodeWithLanguage(enc, text, options = {}) {
         } else if (own && "text" in own) {
             appendRows(at, start);
             let native = own.text;
-            const exact = list(options.lexicon.data.words).some(
-                (entry) => entry.form === lookup || list(entry.variants).includes(lookup),
-            );
+            const exact = statedForms(options.lexicon).has(lookup);
             if (!exact && match[0] !== match[0].toLowerCase()) {
                 const capital =
                     match[0] === initialCapital(match[0].toLowerCase()) ?

package/engine/encoding-lexicon.mjs

@@ -389,9 +389,21 @@ export function nativeRows(records) {
     return rows;
 }
 
-/** Project only conversion-readable language data to a JSON-safe object. */
+/** Each lexicon's projection, made once and shared by every conversion that reads it. */
+const serialized = new WeakMap();
+
+/**
+ * Project only conversion-readable language data to a JSON-safe object. The
+ * projection of one lexicon is made once and shared, so it must not be modified.
+ */
 export function serializeLexicon(lexicon) {
     if (!lexicon) return undefined;
+    if (!serialized.has(lexicon)) serialized.set(lexicon, projectLexicon(lexicon));
+    return serialized.get(lexicon);
+}
+
+/** The conversion-readable data of a lexicon, copied. */
+function projectLexicon(lexicon) {
     const select = (value, keys) =>
         Object.fromEntries(
             keys

package/engine/heading-attributes.mjs

@@ -34,15 +34,12 @@
  */
 
 import { parseExtensionAttributes, refusedAttributes } from "./extension-attributes.mjs";
-import MarkdownIt from "markdown-it";
-
-/** Literal block ranges use the renderer's Markdown grammar. */
-const structureParser = new MarkdownIt({ html: true });
+import { blockTokens } from "./markdown-blocks.mjs";
 
 /** Body lines occupied by code or raw HTML, including comment examples. */
 function literalHeadingLines(source) {
     const lines = new Set();
-    for (const token of structureParser.parse(String(source ?? ""), {})) {
+    for (const token of blockTokens(source)) {
         if (!token.map || !["fence", "code_block", "html_block"].includes(token.type)) continue;
         for (let line = token.map[0]; line < token.map[1]; line++) lines.add(line);
     }

package/engine/language-writings.mjs

@@ -80,8 +80,13 @@ function oneOf(form, candidates) {
     return { ambiguous: { form, glosses: candidates.map(({ gloss }) => gloss) } };
 }
 
+/** Each language data's writings, by encoding and face. */
+const writingsFound = new WeakMap();
+
 /**
- * The writings of a language's words under one encoding in one face.
+ * The writings of a language's words under one encoding in one face. One
+ * `data` yields one set of writings per encoding and face, and each word's
+ * writing is found once, so `data` must not be modified once it is read here.
  *
  * @param {Record<string, any>|undefined} data - The language note's `data`.
  * @param {string} enc - The encoding's name.
@@ -90,6 +95,34 @@ function oneOf(form, candidates) {
  *   The writing of a run's word, and of the `data.words` entry at an index.
  */
 export function writingsOf(data, enc, font) {
+    if (!isMap(data)) return findWritings(data, enc, font);
+    if (!writingsFound.has(data)) writingsFound.set(data, new Map());
+    const byFace = writingsFound.get(data);
+    const key = JSON.stringify([enc, font ?? null]);
+    if (!byFace.has(key)) {
+        const found = findWritings(data, enc, font);
+        const words = new Map();
+        byFace.set(key, {
+            word: (written) => {
+                if (!words.has(written)) words.set(written, found.word(written));
+                return words.get(written);
+            },
+            entry: found.entry,
+        });
+    }
+    return byFace.get(key);
+}
+
+/**
+ * The writings of a language's words under one encoding in one face, found
+ * afresh; see {@link writingsOf}.
+ *
+ * @param {Record<string, any>|undefined} data - The language note's `data`.
+ * @param {string} enc - The encoding's name.
+ * @param {string|undefined} font - The face.
+ * @returns {{word: (word: string) => Writing, entry: (index: number) => Writing}}
+ */
+function findWritings(data, enc, font) {
     const words = listOf(data?.words);
     const affixes = listOf(data?.affixes).filter(isMap);
     const grammar = isMap(data?.grammar) ? data.grammar : undefined;

package/engine/markdown-blocks.mjs

@@ -0,0 +1,58 @@
+/*
+ * This file is part of the Song of Heroic Lands (SoHL) system for Foundry VTT.
+ * Copyright (c) 2024-2026 Tom Rodriguez ("Toasty") — <toasty@heroiclands.org>
+ *
+ * This work is licensed under the GNU General Public License v3.0 (GPLv3).
+ * You may copy, modify, and distribute it under the terms of that license.
+ *
+ * For full terms, see the LICENSE.md file in the project root or visit:
+ * https://www.gnu.org/licenses/gpl-3.0.html
+ *
+ * SPDX-License-Identifier: GPL-3.0-or-later
+ */
+
+/**
+ * The block structure of a Markdown source, in the renderer's grammar.
+ *
+ * Several scanners ask only where a source's fences, code blocks, raw HTML and
+ * quotes lie, each reading block tokens' `type`, `map` and `level` and nothing
+ * else. Inline parsing fills only an `inline` token's `children`, so this
+ * parser skips it, and the block tokens it yields are the ones a full parse
+ * yields. The scanners commonly ask about the same source in turn, so the most
+ * recent sources' tokens are kept.
+ *
+ * @module
+ */
+
+import MarkdownIt from "markdown-it";
+
+/** The renderer's block grammar, with inline parsing and its passes switched off. */
+const parser = new MarkdownIt({ html: true });
+parser.core.ruler.disable(["inline", "linkify", "replacements", "smartquotes", "text_join"]);
+
+/** How many recent sources keep their tokens. */
+const RECENT = 16;
+
+/** @type {Map<string, import("markdown-it").Token[]>} */
+const recent = new Map();
+
+/**
+ * The block tokens of a Markdown source. The array is shared between callers
+ * and must not be modified; an `inline` token carries no `children`.
+ *
+ * @param {unknown} source - Markdown.
+ * @returns {readonly import("markdown-it").Token[]} The block tokens, in order.
+ */
+export function blockTokens(source) {
+    const text = String(source ?? "");
+    const hit = recent.get(text);
+    if (hit) {
+        recent.delete(text);
+        recent.set(text, hit);
+        return hit;
+    }
+    const tokens = parser.parse(text, {});
+    recent.set(text, tokens);
+    if (recent.size > RECENT) recent.delete(recent.keys().next().value);
+    return tokens;
+}

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@heroiclands/package-build",
-    "version": "25.1.1",
+    "version": "25.1.2",
     "description": "Shared toolchain for building and shipping a HeroicLands Foundry VTT package \u2014 content compilation, manifest, localization, staging, bundle, release and deployment.",
     "license": "GPL-3.0-or-later",
     "type": "module",

package/types/engine/content-index.d.mts

@@ -225,10 +225,12 @@ export function resolveCultures(records: Array<Record<string, any>>, { contentPa
  * @param {Map<string, {raw: string}>} [options.sources] - Selected captured source strings.
  * @param {import("./content-session.mjs").ContentSyntax} [options.syntax] - Session scanner projections.
  * @param {Iterable<string>} [options.nameTypes] - Additional bare name kinds from configured type declarations.
+ * @param {ReturnType<typeof import("./content-timings.mjs").createContentTimings>} [options.timings] -
+ *   Invocation stage accounting, which records each phase of the derivation.
  * @returns {Array<Record<string, any>>} The records, in a total order that does
  *   not depend on directory-read order.
  */
-export function collectContentIndex(contentBase: string, { contentPackage, currentDate, defaultCalendar, configFile, rootDir, languages, skipDirectories, assetsBase, manifest, problems, addressContext, foreignRecord, parsed, sources, syntax, nameTypes, }: {
+export function collectContentIndex(contentBase: string, { contentPackage, currentDate, defaultCalendar, configFile, rootDir, languages, skipDirectories, assetsBase, manifest, problems, addressContext, foreignRecord, parsed, sources, syntax, nameTypes, timings, }: {
     contentPackage: string;
     skipDirectories: readonly string[];
     assetsBase?: string | undefined;
@@ -257,6 +259,17 @@ export function collectContentIndex(contentBase: string, { contentPackage, curre
     }> | undefined;
     syntax?: import("./content-session.mjs").ContentSyntax | undefined;
     nameTypes?: Iterable<string> | undefined;
+    timings?: {
+        sync<T>(name: string, work: () => T): T;
+        measure<T>(name: string, work: () => Promise<T> | T): Promise<T>;
+        start(name: string): () => void;
+        hit(name: string): void;
+        snapshot(): {
+            totalMs: number;
+            stages: any;
+            hits: any;
+        };
+    } | undefined;
 }): Array<Record<string, any>>;
 /**
  * Serialize records as JSON Lines.
@@ -301,9 +314,11 @@ export function serializeContentIndex(records: Array<Record<string, any>>): stri
  * @param {import("./content-session.mjs").ContentSyntax} [opts.syntax] - Session scanner projections.
  * @param {Iterable<string>} [opts.nameTypes] - Bare name kinds, defaulting to configured item types.
  * @param {Map<string, object>} [opts.foreignIndex] - Captured dependency address view.
+ * @param {ReturnType<typeof import("./content-timings.mjs").createContentTimings>} [opts.timings] -
+ *   Invocation stage accounting.
  * @returns {object[]} One record per note, plus one per documentation entry.
  */
-export function indexRecordsFor({ contentBase, assetsBase, config, skipDirectories, problems, parsed, sources, syntax, nameTypes, foreignIndex, }?: {
+export function indexRecordsFor({ contentBase, assetsBase, config, skipDirectories, problems, parsed, sources, syntax, nameTypes, foreignIndex, timings, }?: {
     contentBase?: string | undefined;
     assetsBase?: string | undefined;
     config?: object | undefined;
@@ -322,6 +337,17 @@ export function indexRecordsFor({ contentBase, assetsBase, config, skipDirectori
     syntax?: import("./content-session.mjs").ContentSyntax | undefined;
     nameTypes?: Iterable<string> | undefined;
     foreignIndex?: Map<string, object> | undefined;
+    timings?: {
+        sync<T>(name: string, work: () => T): T;
+        measure<T>(name: string, work: () => Promise<T> | T): Promise<T>;
+        start(name: string): () => void;
+        hit(name: string): void;
+        snapshot(): {
+            totalMs: number;
+            stages: any;
+            hits: any;
+        };
+    } | undefined;
 }): object[];
 /**
  * Prepare the complete bodies a content-index publication describes.

package/types/engine/content-session.d.mts

@@ -27,6 +27,7 @@ export function createContentSession({ config, contentBase, assetsBase, skipDire
     timings?: {
         sync<T>(name: string, work: () => T): T;
         measure<T>(name: string, work: () => Promise<T> | T): Promise<T>;
+        start(name: string): () => void;
         hit(name: string): void;
         snapshot(): {
             totalMs: number;

package/types/engine/content-timings.d.mts

@@ -4,6 +4,13 @@ export function createContentTimings(): {
     sync<T>(name: string, work: () => T): T;
     /** @template T @param {string} name @param {() => Promise<T>|T} work @returns {Promise<T>} */
     measure<T>(name: string, work: () => Promise<T> | T): Promise<T>;
+    /**
+     * Open a stage that runs until the returned function is called, or
+     * until the stage it was opened in ends.
+     * @param {string} name
+     * @returns {() => void}
+     */
+    start(name: string): () => void;
     /** @param {string} name */
     hit(name: string): void;
     snapshot(): {

package/types/engine/encoding-lexicon.d.mts

@@ -59,24 +59,11 @@ export function languageNoteFindings(config: {
  *   word's 0-based place in `data.words`, `native` the form's in its `native`.
  */
 export function nativeRows(records: Iterable<Record<string, any>>): Array<Record<string, string | number | null>>;
-/** Project only conversion-readable language data to a JSON-safe object. */
-export function serializeLexicon(lexicon: any): {
-    language: any;
-    data: {
-        grammar?: {
-            [k: string]: any;
-        } | undefined;
-        words: {
-            [k: string]: any;
-        }[];
-        affixes: {
-            [k: string]: any;
-        }[];
-        register: {
-            [k: string]: any;
-        }[];
-    };
-} | undefined;
+/**
+ * Project only conversion-readable language data to a JSON-safe object. The
+ * projection of one lexicon is made once and shared, so it must not be modified.
+ */
+export function serializeLexicon(lexicon: any): any;
 /**
  * Resolve encoding, face and optional language input without loading files.
  *
@@ -129,23 +116,7 @@ export function encodingFor(ctx: import("./content-scripts.mjs").RunContext, { e
         };
     }>;
     font: any;
-    lexicon: {
-        language: any;
-        data: {
-            grammar?: {
-                [k: string]: any;
-            } | undefined;
-            words: {
-                [k: string]: any;
-            }[];
-            affixes: {
-                [k: string]: any;
-            }[];
-            register: {
-                [k: string]: any;
-            }[];
-        };
-    } | undefined;
+    lexicon: any;
 } | undefined;
 /** Why a face is incompatible with the selected encoding. */
 export function fontProblem(encoding: any, font: any): string | undefined;

package/types/engine/language-writings.d.mts

@@ -16,7 +16,9 @@
  */
 export function nativeText(natives: unknown, enc: string, font: string | undefined): string | undefined;
 /**
- * The writings of a language's words under one encoding in one face.
+ * The writings of a language's words under one encoding in one face. One
+ * `data` yields one set of writings per encoding and face, and each word's
+ * writing is found once, so `data` must not be modified once it is read here.
  *
  * @param {Record<string, any>|undefined} data - The language note's `data`.
  * @param {string} enc - The encoding's name.

package/types/engine/markdown-blocks.d.mts

@@ -0,0 +1,8 @@
+/**
+ * The block tokens of a Markdown source. The array is shared between callers
+ * and must not be modified; an `inline` token carries no `children`.
+ *
+ * @param {unknown} source - Markdown.
+ * @returns {readonly import("markdown-it").Token[]} The block tokens, in order.
+ */
+export function blockTokens(source: unknown): readonly import("markdown-it").Token[];