@heroiclands/package-build 23.8.0 → 23.10.0

package/engine/content-links.mjs

@@ -59,6 +59,7 @@ import { cultureRecordProblems, cultureInheritedFields } from "./culture-parents
 import path from "node:path";
 
 import { matchAllOutsideCode } from "./code-fences.mjs";
+import { splitFrontmatter } from "./diagnostics.mjs";
 import { expandContentTables } from "./content-tables.mjs";
 import { collectAnchors } from "./anchors.mjs";
 // The corpus, and everything derived from it, read from the one place that
@@ -90,7 +91,7 @@ import {
 import { loadForeignIndexes, noContentIndexPackages } from "./metadata-index.mjs";
 import { frontmatterWikilinks, slugify } from "./web-wikilinks.mjs";
 import { homepageAddresses, isHomepage } from "./homepage.mjs";
-import { parseWikilink, WIKILINK } from "./wikilink-syntax.mjs";
+import { isEventAnchor, parseWikilink, WIKILINK } from "./wikilink-syntax.mjs";
 import { readQualifier } from "./wikilinks.mjs";
 
 /**
@@ -129,13 +130,11 @@ export function anchorsOf(body) {
  * not hypothetical: this module carried its own anchor reader until the anchor
  * anchor half, and the two disagreed about which anchors existed.
  *
- * **The file is opened for its bytes and nothing else.** The index deliberately
- * carries no note *body*, and a link lives in the body — so each note is read
- * once, here, for the prose. Everything *about* the note — its frontmatter, its
- * addresses, its anchors — is already in the record, and none of it is derived
- * a second time. That is one read per note rather than the two this module did
- * before, since the walk read the file and it then read it again for the raw
- * text.
+ * **Raw sources can come from the caller's capture.** The published index
+ * carries no note body, while a link lives in the prose. A caller that already
+ * captured the sources supplies `parsed`, so this pass uses those exact raw
+ * strings. Without a captured string, it opens the file for its bytes.
+ * Frontmatter, addresses and anchors still come from the records.
  *
  * The index holds each compiled document under its complete Address. Readable
  * content uses `note`, and system documents use their declared system.
@@ -153,14 +152,27 @@ export function anchorsOf(body) {
  * @param {object[]} [opts.records] - Index records the caller already derived,
  *   so a command that also needs them — every one of them does, to answer its
  *   `sql` tables — enumerates the corpus once rather than twice.
+ * @param {Map<string, {raw: string}>} [opts.parsed] - Sources captured alongside
+ *   `records` in this invocation, keyed by absolute note path. A raw string is
+ *   used verbatim; a missing string falls back to reading the file. The caller
+ *   owns the map, which this function only reads.
  * @param {object[]} [opts.problems] - Collects the notes the index cannot
  *   record, as diagnostics, instead of letting one of them abort the check
  *   before it has reported anything else.
+ * @param {ReturnType<import("./metadata-index.mjs").loadForeignIndexes>} [opts.foreign] - Captured dependency address and reference views.
  * @returns {object} The notes, the index, and the resolvers built over it.
  */
 export function buildLinkIndex(
     contentBase,
-    { config, skipDirectories, sqlTables, records, problems } = {},
+    {
+        config,
+        skipDirectories,
+        sqlTables,
+        records,
+        parsed,
+        problems,
+        foreign: capturedForeign,
+    } = {},
 ) {
     const notes = [];
     const frontmatterLinks = [];
@@ -221,8 +233,10 @@ export function buildLinkIndex(
         const absPath = noteFile(contentBase, record);
         // The raw text is kept beside the parsed body: a consumer's own checks
         // may need what frontmatter carried, which the body has dropped.
-        const raw = fs.readFileSync(absPath, "utf8");
-        const body = raw.replace(/^---\n[\s\S]*?\n---\n?/, "");
+        const capturedRaw = parsed?.get(absPath)?.raw;
+        const raw =
+            typeof capturedRaw === "string" ? capturedRaw : fs.readFileSync(absPath, "utf8");
+        const body = splitFrontmatter(raw)?.body ?? raw;
         const type = record.type.toLowerCase();
         const note = { file: absPath, rel, fm, body, raw, type };
         const hierarchyProblems = cultureRecordProblems(record);
@@ -243,7 +257,10 @@ export function buildLinkIndex(
         // same headings — the disagreement the anchor half removes.
         anchors.set(
             note,
-            new Set((record.anchors ?? []).filter((a) => a.kind === "prose").map((a) => a.slug)),
+            new Set([
+                ...(record.anchors ?? []).filter((a) => a.kind === "prose").map((a) => a.slug),
+                ...(sqlTables?.get(absPath)?.anchors ?? []).map((a) => a.slug),
+            ]),
         );
         eventAnchorsOf.set(
             note,
@@ -292,9 +309,10 @@ export function buildLinkIndex(
     // is never checked at all.
     const localPackages = new Set([pkg]);
     const foreign =
-        config ?
+        capturedForeign ??
+        (config ?
             loadForeignIndexes(config, localPackages)
-        :   { index: new Map(), packages: new Set(), stale: [] };
+        :   { index: new Map(), packages: new Set(), stale: [] });
     for (const v of foreign.index.values()) if (v.type) types.add(v.type);
 
     // A stub counts. It publishes no page, so nothing may *link* to one, but its
@@ -331,7 +349,13 @@ export function buildLinkIndex(
      */
     function expandedBody(note) {
         const body = note.body;
-        if (!/^[ \t]*(?:`{3,}|~{3,})[ \t]*(?:sql|pagelist)\b/im.test(body)) return body;
+        const prepared = sqlTables?.get(note.file);
+        if (
+            !prepared?.generated &&
+            !prepared?.lexical?.length &&
+            !/^[ \t]*(?:`{3,}|~{3,})[ \t]*(?:sql|pagelist)\b/im.test(body)
+        )
+            return body;
         return expandContentTables(body, {
             source: note.file,
             // A generated link is checked like an authored one, so both
@@ -940,18 +964,37 @@ export function auditLinks(index) {
     for (const note of notes) {
         for (const { target, rawTarget, anchor, text, occurrence, labelled } of linksOf(note)) {
             if (!anchor || !labelled) continue;
-            const dest = target ? resolve(target) : note;
-            // An unresolvable target is reported by the pass below; its anchor
-            // has nothing to be checked against. Neither has a file: an asset
-            // resolves but has no body, so it declares the empty set of anchors
-            // and every `#section` on one is dead.
-            if (!dest) continue;
-            if (!(anchors.get(dest) ?? new Set()).has(slugify(anchor))) {
+            const local = target ? resolve(target) : note;
+            const foreign = !local && target ? manifestHit(target) : null;
+            // An unresolved address is reported below. A foreign sheet opens
+            // its system document, so only the readable note view names prose
+            // sections. A public note's GM target is an audience refusal below.
+            if (!local) {
+                if (!foreign || (foreign.gm && !isGmNote(note.fm))) continue;
+                const qualified = readQualifier(
+                    target,
+                    index.types,
+                    index.packages,
+                    index.noIndexPackages,
+                );
+                if (qualified?.system !== NOTE_SYSTEM) continue;
+            }
+            const dest = local ?? foreign;
+            const slug = local ? slugify(anchor) : anchor;
+            const event =
+                local ?
+                    (index.eventAnchorsOf?.get(local) ?? new Set()).has(anchor)
+                :   isEventAnchor(foreign, anchor);
+            // Fetched note anchors are the publisher's declaration, independent
+            // of a site URL or a Foundry UUID. An asset has no body and therefore
+            // declares no sections; an absent foreign anchor list does too.
+            const declared =
+                local ?
+                    (anchors.get(local) ?? new Set()).has(slug)
+                :   foreign.noteAnchors?.some((one) => one.slug === anchor && one.kind === "prose");
+            if (!declared || (!local && event)) {
                 deadAnchors.push({
-                    reason:
-                        (index.eventAnchorsOf?.get(dest) ?? new Set()).has(anchor) ?
-                            "event-anchor"
-                        :   "unknown-anchor",
+                    reason: event ? "event-anchor" : "unknown-anchor",
                     note,
                     link: `${rawTarget}#${anchor}`,
                     // The two the shared `unknown-anchor` wording reads, in the
@@ -960,7 +1003,7 @@ export function auditLinks(index) {
                     // declare. Carrying them is what lets one sentence serve
                     // the checker and the build.
                     target: rawTarget,
-                    anchor: slugify(anchor),
+                    anchor: slug,
                     dest,
                     text,
                     occurrence,

package/engine/content-session.mjs

@@ -0,0 +1,247 @@
+/*
+ * This file is part of the Song of Heroic Lands (SoHL) system for Foundry VTT.
+ * Copyright (c) 2024-2026 Tom Rodriguez ("Toasty") — <toasty@heroiclands.org>
+ *
+ * This work is licensed under the GNU General Public License v3.0 (GPLv3).
+ * You may copy, modify, and distribute it under the terms of that license.
+ *
+ * For full terms, see the LICENSE.md file in the project root or visit:
+ * https://www.gnu.org/licenses/gpl-3.0.html
+ *
+ * SPDX-License-Identifier: GPL-3.0-or-later
+ */
+
+/** One invocation's authored sources and context-specific prepared views. */
+import path from "node:path";
+import * as contentIndex from "./content-index.mjs";
+import * as blocks from "./content-blocks.mjs";
+import * as alerts from "./content-alerts.mjs";
+import * as spans from "./content-spans.mjs";
+import * as sql from "./sql-tables.mjs";
+import { captureContentSources } from "./content-sources.mjs";
+import { buildLinkIndex } from "./content-links.mjs";
+import { buildContentLinkIndex } from "./helpers.mjs";
+import { loadForeignIndexes } from "./metadata-index.mjs";
+import { loadPackConfig } from "./pack-config.mjs";
+import { routerFor } from "./pack-router.mjs";
+import { lintNoteStates } from "./stub-lint.mjs";
+
+/**
+ * @typedef {"index"|"lint"|"links"|"compile"} ContentTarget
+ * @typedef {(source: string) => {blocks: ReturnType<typeof blocks.scanBlocks>, alerts: ReturnType<typeof alerts.scanAlerts>, spans: ReturnType<typeof spans.scanSpans>}} ContentSyntax
+ * @typedef {{config?: import("../content-config.mjs").ContentBuildConfig, contentBase?: string, assetsBase?: string, skipDirectories?: readonly string[], router?: ReturnType<typeof routerFor>}} ContentSessionContext
+ * @typedef {{frontmatter: object|null, body: string, raw: string, bodyLine?: number, bodyColumn?: number}} ContentNoteSource
+ * @typedef {object} ContentSession
+ * @property {import("../content-config.mjs").ContentBuildConfig} config
+ * @property {string} contentBase
+ * @property {string} assetsBase
+ * @property {readonly string[]} skipDirectories
+ * @property {ReturnType<typeof routerFor>} router
+ * @property {ReturnType<typeof captureContentSources>} inventory
+ * @property {Map<string, {raw: string}>} sources
+ * @property {object[]} records
+ * @property {Map<string, ContentNoteSource>} parsed
+ * @property {object[]} problems
+ * @property {ReturnType<typeof loadForeignIndexes>} foreign
+ * @property {ContentSyntax} syntax
+ * @property {(options?: ContentSessionContext) => void} assertContext
+ * @property {(audience?: "all"|"public") => ReturnType<typeof sql.prepareTreeSqlTables>} sqlTables
+ * @property {() => ReturnType<typeof sql.prepareLinkSqlTables>} linkTables
+ * @property {(kind?: "lint"|"links") => Promise<ReturnType<typeof buildLinkIndex>>} audit
+ * @property {() => Promise<import("./content-index.mjs").PreparedContentIndex>} publication
+ * @property {() => ReturnType<import("./compile-corpus.mjs").buildCompileCorpus>} compileCorpus
+ * @property {(targets: readonly ContentTarget[]) => Promise<ContentSession>} prepare
+ * @property {() => void} dispose
+ */
+
+/**
+ * The explicit runner's target vocabulary and execution order.
+ * @type {readonly ContentTarget[]}
+ */
+export const CONTENT_TARGETS = Object.freeze(["index", "lint", "links", "compile"]);
+
+/**
+ * Capture one source tree for index, lint, links and compilation.
+ * No source or parser state is persisted or reused by another invocation.
+ * Prepared SQL databases close within their existing preparation functions.
+ * @param {object} [options]
+ * @param {import("../content-config.mjs").ContentBuildConfig} [options.config] - Exact resolved configuration.
+ * @param {string} [options.contentBase] - Content root, defaulting to configuration.
+ * @param {string} [options.assetsBase] - Assets root, defaulting to configuration.
+ * @param {readonly string[]} [options.skipDirectories] - Resolved content scope.
+ * @param {ReturnType<typeof routerFor>} [options.router] - One router shared by this session's compile.
+ * @returns {Promise<ContentSession>} Captured sources and lazy prepared views; dispose after use.
+ */
+export async function createContentSession({
+    config = loadPackConfig(),
+    contentBase = config.paths.content,
+    assetsBase = config.paths.assets,
+    skipDirectories = config.skipDirectories,
+    router = routerFor(config),
+} = {}) {
+    const tree = path.resolve(contentBase);
+    const assets = path.resolve(assetsBase);
+    const scope = [...skipDirectories];
+    const inventory = captureContentSources(tree, assets, scope);
+    const parsed = new Map();
+    const problems = [];
+    const foreign = loadForeignIndexes(config, [config.contentPackage]);
+    const syntaxCache = new Map();
+    const prepared = new Map();
+    let disposed = false;
+    const active = () => {
+        if (disposed) throw new Error("Content session has been disposed");
+    };
+    const syntax = (source) => {
+        active();
+        const text = String(source ?? "");
+        if (!syntaxCache.has(text))
+            syntaxCache.set(text, {
+                blocks: blocks.scanBlocks(text),
+                alerts: alerts.scanAlerts(text),
+                spans: spans.scanSpans(text),
+            });
+        return syntaxCache.get(text);
+    };
+    const records = contentIndex.indexRecordsFor({
+        config,
+        contentBase: tree,
+        assetsBase: assets,
+        skipDirectories: scope,
+        sources: inventory.indexSources,
+        parsed,
+        problems,
+        foreignIndex: foreign.index,
+        syntax,
+    });
+    const memo = (key, make) => {
+        active();
+        if (!prepared.has(key)) prepared.set(key, Promise.resolve().then(make));
+        return prepared.get(key);
+    };
+    const session = {
+        config,
+        contentBase: tree,
+        assetsBase: assets,
+        skipDirectories: scope,
+        router,
+        inventory,
+        sources: inventory.sources,
+        records,
+        parsed,
+        problems,
+        foreign,
+        syntax,
+        /** Refuse use with a different configuration, root, scope or router. */
+        assertContext(options = {}) {
+            active();
+            if (
+                (options.config !== undefined && options.config !== config) ||
+                (options.contentBase !== undefined && path.resolve(options.contentBase) !== tree) ||
+                (options.assetsBase !== undefined && path.resolve(options.assetsBase) !== assets) ||
+                (options.skipDirectories !== undefined &&
+                    JSON.stringify(options.skipDirectories) !== JSON.stringify(scope)) ||
+                (options.router !== undefined && options.router !== router)
+            )
+                throw new Error(
+                    "Content session context does not match configuration, roots, scope or router",
+                );
+        },
+        async sqlTables(audience = "all") {
+            if (audience !== "all" && audience !== "public")
+                throw new Error(`Unknown SQL audience: ${audience}`);
+            return memo(`sql:${audience}`, () =>
+                sql.prepareTreeSqlTables(tree, {
+                    config,
+                    records,
+                    parsed,
+                    audience,
+                    skipDirectories: scope,
+                    foreignIndex: foreign.index,
+                    syntax,
+                }),
+            );
+        },
+        async linkTables() {
+            return memo("link-tables", () =>
+                sql.prepareLinkSqlTables(tree, {
+                    config,
+                    records,
+                    parsed,
+                    skipDirectories: scope,
+                    foreignIndex: foreign.index,
+                    prepareAudience: (audience) => session.sqlTables(audience),
+                }),
+            );
+        },
+        async audit(kind = "links") {
+            if (kind !== "lint" && kind !== "links") throw new Error(`Unknown audit view: ${kind}`);
+            return memo(`audit:${kind}`, async () =>
+                buildLinkIndex(tree, {
+                    config,
+                    records,
+                    parsed,
+                    skipDirectories: scope,
+                    foreign,
+                    sqlTables:
+                        kind === "lint" ?
+                            await session.sqlTables("all")
+                        :   await session.linkTables(),
+                }),
+            );
+        },
+        async publication() {
+            return memo("publication", () =>
+                contentIndex.prepareContentIndex({ config, contentBase: tree, session }),
+            );
+        },
+        async compileCorpus() {
+            return memo("compile", async () => {
+                const sqlTables = await session.sqlTables("all");
+                return {
+                    records,
+                    parsed,
+                    problems,
+                    sqlTables,
+                    syntax,
+                    linkIndex: buildContentLinkIndex(tree, router, {
+                        config,
+                        records,
+                        sqlTables,
+                        foreign,
+                    }),
+                };
+            });
+        },
+        /** Complete selected preparations before a caller starts a writer. */
+        async prepare(targets) {
+            active();
+            for (const target of targets)
+                if (!CONTENT_TARGETS.includes(target))
+                    throw new Error(`Unknown content target: ${target}`);
+            if (targets.includes("index") || targets.includes("compile")) {
+                const stateErrors = lintNoteStates(tree, {
+                    records,
+                    parsed,
+                    contentPackage: config.contentPackage,
+                }).findings.filter((finding) => finding.severity === "error");
+                const findings = [...problems, ...stateErrors];
+                if (findings.length)
+                    throw Object.assign(new Error("Content preparation failed"), { findings });
+            }
+            for (const target of CONTENT_TARGETS.filter((target) => targets.includes(target))) {
+                if (target === "index") await session.publication();
+                else if (target === "compile") await session.compileCorpus();
+                else await session.audit(target);
+            }
+            return session;
+        },
+        /** Release this invocation's derived views; captured inputs stay unchanged. */
+        dispose() {
+            disposed = true;
+            prepared.clear();
+            syntaxCache.clear();
+        },
+    };
+    return session;
+}

package/engine/content-sources.mjs

@@ -0,0 +1,110 @@
+/*
+ * This file is part of the Song of Heroic Lands (SoHL) system for Foundry VTT.
+ * Copyright (c) 2024-2026 Tom Rodriguez ("Toasty") — <toasty@heroiclands.org>
+ *
+ * This work is licensed under the GNU General Public License v3.0 (GPLv3).
+ * You may copy, modify, and distribute it under the terms of that license.
+ *
+ * For full terms, see the LICENSE.md file in the project root or visit:
+ * https://www.gnu.org/licenses/gpl-3.0.html
+ *
+ * SPDX-License-Identifier: GPL-3.0-or-later
+ */
+
+/** Physical source capture, independent of whether a note can be indexed. */
+import fs from "node:fs";
+import path from "node:path";
+import log from "loglevel";
+import { DEPLOYED_ASSET_TYPES } from "./asset-types.mjs";
+
+const TEXT_EXTENSIONS = new Set([".md", ".markdown", ".yaml", ".yml", ".json"]);
+
+/**
+ * Capture exact text once and retain directory entries for physical checks.
+ * The index walk and raw check walks retain their established traversal orders.
+ * @param {string} contentBase - Absolute content root.
+ * @param {string} assetsBase - Absolute assets root.
+ * @param {readonly string[]} skipDirectories - Resolved directory scope.
+ * @returns {{sources: Map<string, {raw: string}>, indexSources: Map<string, {raw: string}>, paths: Array<{root: string, file: string, name: string, directory: boolean}>}} Captured input and physical walk order.
+ */
+export function captureContentSources(contentBase, assetsBase, skipDirectories) {
+    const sources = new Map();
+    const directories = new Map();
+    const readErrors = new Map();
+    const paths = [];
+    const skip = new Set(skipDirectories);
+    const walk = (dir, text, includeHidden = false, root = dir) => {
+        if (!fs.existsSync(dir)) return;
+        let entries;
+        try {
+            entries = fs.readdirSync(dir, { withFileTypes: true });
+        } catch (error) {
+            if (text) log.warn(`Cannot read directory ${dir}: ${error.message}`);
+            return;
+        }
+        if (text) directories.set(dir, entries);
+        for (const entry of entries) {
+            if (
+                (!includeHidden && entry.name.startsWith(".")) ||
+                (entry.isDirectory() && skip.has(entry.name))
+            )
+                continue;
+            const file = path.join(dir, entry.name);
+            paths.push({ root, file, name: entry.name, directory: entry.isDirectory() });
+            if (entry.isDirectory()) walk(file, text, includeHidden, root);
+            else if (text && TEXT_EXTENSIONS.has(path.extname(entry.name).toLowerCase())) {
+                try {
+                    sources.set(file, { raw: fs.readFileSync(file, "utf8") });
+                } catch (error) {
+                    // Selected Markdown must retain the index reader's refusal.
+                    // Only the independent raw-checker inputs skip read errors.
+                    if (entry.isFile() && entry.name.endsWith(".md")) readErrors.set(file, error);
+                }
+            }
+        }
+    };
+    // The index scope includes dot paths; individual raw checks
+    // filter those paths through capturedSources below.
+    walk(contentBase, true, true);
+    for (const root of new Set(
+        DEPLOYED_ASSET_TYPES.map((entry) => path.resolve(assetsBase, entry.root)),
+    ))
+        if (root !== path.resolve(contentBase)) walk(root, false);
+    const indexSources = new Map();
+    const stack = [contentBase];
+    while (stack.length) {
+        const dir = stack.pop();
+        for (const entry of directories.get(dir) ?? []) {
+            if (entry.isDirectory() && skip.has(entry.name)) continue;
+            const file = path.join(dir, entry.name);
+            if (entry.isDirectory()) stack.push(file);
+            else if (readErrors.has(file)) throw readErrors.get(file);
+            else if (entry.isFile() && entry.name.endsWith(".md") && sources.has(file))
+                indexSources.set(file, sources.get(file));
+        }
+    }
+    return { sources, indexSources, paths };
+}
+
+/**
+ * Select the exact physical scope of a raw checker from a captured map.
+ * @param {string} contentBase - Content root.
+ * @param {Map<string, {raw: string}>} sources - Captured physical sources.
+ * @param {readonly string[]} skipDirectories - Directory names to ignore.
+ * @param {Set<string>} extensions - Lowercase extensions accepted by the caller.
+ */
+export function* capturedSources(contentBase, sources, skipDirectories, extensions) {
+    const root = path.resolve(contentBase);
+    for (const [file, source] of sources) {
+        const relative = path.relative(root, file);
+        if (relative.startsWith(`..${path.sep}`) || path.isAbsolute(relative)) continue;
+        if (
+            relative
+                .split(path.sep)
+                .some((name) => name.startsWith(".") || skipDirectories.includes(name))
+        )
+            continue;
+        if (extensions.has(path.extname(file).toLowerCase()) && typeof source.raw === "string")
+            yield [file, source.raw];
+    }
+}