@heroiclands/package-build 17.1.0 → 18.0.0

package/engine/generate.mjs

@@ -46,14 +46,35 @@ import { foreignItemCatalogDirs } from "./foreign-catalog.mjs";
 import { Items } from "../sohl/items.mjs";
 import { Journals } from "./journals.mjs";
 import { Actors } from "../sohl/actors.mjs";
+import { Hm3Items } from "../hm3/items.mjs";
+import { Hm3Actors } from "../hm3/actors.mjs";
 import { Macros } from "./macros.mjs";
 import { Scenes } from "./scenes.mjs";
-import { statsForPack, loadFolders, buildFolderResolver, writeFolderDocs } from "./helpers.mjs";
+import { Bundles } from "./bundles.mjs";
+import {
+    statsForPack,
+    loadFolders,
+    buildFolderResolver,
+    writeFolderDocs,
+    parseMarkdownFile,
+    folderFilename,
+} from "./helpers.mjs";
+import {
+    buildFolderNoteIndex,
+    collectFolderNotes,
+    FOLDER_TYPE,
+    folderDocument,
+} from "./folder-notes.mjs";
 import { countContentNotes } from "./content-tree.mjs";
 import { emitDiagnostic } from "./diagnostics.mjs";
+// The corpus every pass runs over, derived once (#243).
+import { buildCompileCorpus } from "./compile-corpus.mjs";
+// The record accessors only — see `engine/index-records.mjs` (#243).
+import { isNoteRecord, noteFile } from "./index-records.mjs";
 import { loadPackConfig } from "./pack-config.mjs";
 import { routerFor } from "./pack-router.mjs";
 import { unclaimedNoteFindings } from "./note-claims.mjs";
+import { contentPackage } from "./content-package.mjs";
 
 /**
  * The compiler class for each Foundry document type a pack may hold.
@@ -63,6 +84,15 @@ import { unclaimedNoteFindings } from "./note-claims.mjs";
  * compiler here. Unknown types fail the build rather than defaulting, so a pack
  * declaring a type nothing can compile is loud at the first pass instead of
  * shipping empty.
+ *
+ * **Two of the six are a system's, and the SoHL pair is not a default.** An
+ * Item or an Actor *is* a system's data — both passes declare
+ * `requiresSystemBlock` — so which compiler a pack gets is decided together
+ * with which system it declares; see {@link SYSTEM_COMPILERS}. The four
+ * system-neutral passes have one implementation because a JournalEntry, a
+ * Macro, a Scene and an Adventure are Foundry's documents rather than any
+ * system's — an `Adventure` does not even have a `system` field, which is why a
+ * bundle spanning two systems is two documents (#259).
  */
 const COMPILERS = {
     Item: Items,
@@ -70,8 +100,44 @@ const COMPILERS = {
     Actor: Actors,
     Macro: Macros,
     Scene: Scenes,
+    Adventure: Bundles,
 };
 
+/**
+ * The system-specific compilers, by the system a pack declares (#139).
+ *
+ * A repository feeding two systems declares one Item pack and one Actor pack
+ * per system — `harn-ensemble` has `actors-hm3` and `actors-sohl` — and each
+ * pack's `system:` is what says whose data model its documents are shaped for.
+ * That is the same field the `_stats` stamp, the item-catalogue scope and the
+ * `itemBuilders` lookup already read, so nothing new is declared to make the
+ * compiler follow it.
+ *
+ * A system with no entry — or a pack that declares none — falls back to
+ * {@link COMPILERS}. That keeps every single-system configuration meaning
+ * exactly what it did: SoHL's passes were the only ones, so they stay the
+ * answer where nothing says otherwise.
+ *
+ * @type {Readonly<Record<string, Readonly<Record<string, Function>>>>}
+ */
+const SYSTEM_COMPILERS = Object.freeze({
+    sohl: Object.freeze({ Item: Items, Actor: Actors }),
+    hm3: Object.freeze({ Item: Hm3Items, Actor: Hm3Actors }),
+});
+
+/**
+ * The compiler class a pack of one document type and one system gets.
+ *
+ * @param {string} docType - The Foundry document type the pack holds.
+ * @param {string|null} [system] - The system the pack declares, if any.
+ * @returns {Function|undefined} The compiler class, or `undefined` for a
+ *   document type nothing here compiles — which {@link generatePack} reports
+ *   rather than defaulting past.
+ */
+export function compilerFor(docType, system = null) {
+    return (system && SYSTEM_COMPILERS[system]?.[docType]) || COMPILERS[docType];
+}
+
 /**
  * Root of the build-only JSON tree for one pack.
  *
@@ -122,6 +188,45 @@ export function itemPackJsonDirs(config = loadPackConfig(), system = null) {
         .map((pack) => packJsonDir(pack.name, config));
 }
 
+/**
+ * The compiled JSON a bundle may hold copies of, by document type.
+ *
+ * An `Adventure` carries **copies**, not references, so a bundle resolves its
+ * `contents` against compiled output rather than against the content tree — the
+ * same arrangement the actors pass has for `itemsSourceDirs`, generalised to
+ * every document class an Adventure can hold (#259).
+ *
+ * Two kinds of pack are left out, each because it holds nothing a note
+ * addresses. A **prebuilt** pack's JSON is checked in rather than compiled, so
+ * no note is routed into it and nothing in it answers to an address. An
+ * **Adventure** pack holds Adventures, and Foundry's `contentFields` has no
+ * field for one — a bundle of bundles is not a shape the document admits.
+ *
+ * **Scoped to one system when the pack has one**, exactly as
+ * {@link itemPackJsonDirs} is: a pack declaring `system: sohl` reads that
+ * system's packs and the system-neutral ones, so a `(type, shortcode)` that
+ * exists in two systems is read out of the right catalogue. Asking for no
+ * system reads them all, which is every single-system build.
+ *
+ * @param {object} [config] - The resolved build configuration. Defaults to this
+ *   repository's.
+ * @param {string|null} [system] - The system whose documents are wanted.
+ *   Omitted or `null`, every pack is read.
+ * @returns {Record<string, string[]>} Each readable pack's JSON directory, by
+ *   the Foundry document type it holds.
+ */
+export function bundleSourceJsonDirs(config = loadPackConfig(), system = null) {
+    /** @type {Record<string, string[]>} */
+    const dirs = {};
+    for (const pack of config.packs) {
+        if (pack.prebuilt) continue;
+        if (pack.type === "Adventure") continue;
+        if (system != null && pack.system && pack.system !== system) continue;
+        (dirs[pack.type] ??= []).push(packJsonDir(pack.name, config));
+    }
+    return dirs;
+}
+
 /**
  * The document types whose compiled output a pass of this type reads.
  *
@@ -258,11 +363,13 @@ async function generatePack(
     config,
     router,
     routingReporter,
+    folderNotes,
+    corpus,
 ) {
     const contentBase = config.paths.content;
     const dest = packJsonDir(name, config);
 
-    const packClass = COMPILERS[type];
+    const packClass = compilerFor(type, system ?? null);
     if (!packClass) {
         log.error(
             `Pack ${name}: no compiler for document type "${type}" — the ` +
@@ -274,15 +381,55 @@ async function generatePack(
     log.info(`Pack ${name}: ${contentBase} → ${dest}`);
 
     let folderList;
-    let resolver;
+    let yamlResolver;
     try {
         folderList = folders ? loadFolders(path.join(contentBase, folders)) : [];
-        ({ resolver } = buildFolderResolver(folderList));
+        ({ resolver: yamlResolver } = buildFolderResolver(folderList));
     } catch (err) {
         log.error(`${name} ${folders} validation failed: ${err.message}`);
         return { errors: 1, compiled: 0 };
     }
 
+    // Which folder notes this pack turned out to hold something for. A folder
+    // materialises in every pack holding a document that references it, so the
+    // set is not knowable until the pass has compiled — which is why these
+    // documents are written after `compile()` and the YAML ones before it
+    // (#257).
+    /** @type {Set<import("./folder-notes.mjs").FolderNote>} */
+    const referencedFolders = new Set();
+
+    /**
+     * The Foundry folder id a note names, by address or by id.
+     *
+     * The two spellings resolve against two different sources and always did:
+     * `packFolder` names a folder **note**, resolved through the address index
+     * shared by the whole build, and `folder` names a Foundry **id** declared
+     * in this pack's own YAML. Which one applies is the field the value was
+     * written in, never the string (#251).
+     *
+     * @param {string|null|undefined} value - As authored.
+     * @param {object} [opts]
+     * @param {boolean} [opts.isAddress] - Whether `value` is a folder address.
+     * @returns {string|null} The folder id, or `null` for an absent value.
+     */
+    const resolver = (value, { isAddress = false } = {}) => {
+        if (value == null || value === "") return null;
+        if (!isAddress) return yamlResolver(value);
+        const folder = folderNotes.resolve(value);
+        // Its ancestors with it: a `Folder` whose parent is absent from the
+        // pack is an orphan Foundry renders at the root, so materialising a
+        // folder without its chain breaks the tree at the top rather than
+        // merely leaving it incomplete.
+        referencedFolders.add(folder);
+        // Asked of *this* pack: a folder's hierarchy is per-pack even though
+        // its identity is not, so the chain that has to come with it is the
+        // chain it has here.
+        for (const ancestor of folderNotes.ancestorsOf(folder, name)) {
+            referencedFolders.add(ancestor);
+        }
+        return folder.id;
+    };
+
     // Wipe and recreate so removed content notes leave no stale JSON.
     fs.rmSync(dest, { recursive: true, force: true });
     fs.mkdirSync(dest, { recursive: true });
@@ -304,6 +451,8 @@ async function generatePack(
     const pack = new packClass({
         contentBase,
         dest,
+        // The corpus this compile derived once, shared by every pass (#243).
+        corpus,
         companionDests,
         // The actors pass resolves each being's embedded items against the items
         // passes' output. That used to be an unwritten sibling-directory contract
@@ -320,7 +469,15 @@ async function generatePack(
         // cache throws naming `content-build deps fetch` rather than
         // downloading inside a compile.
         foreignSourceDirs: foreignItemCatalogDirs(config),
+        // The bundles pass resolves each Adventure's members against the output
+        // of every pass that produces one. Stated from the configured pack list
+        // for the same reason `itemsSourceDirs` is (#1508), and scoped to this
+        // pack's system so a bundle holds the catalogue it is compiled for.
+        bundleSourceDirs: bundleSourceJsonDirs(config, system ?? null),
         folderResolver: resolver,
+        // One answer to "which files are the corpus?", from the configuration
+        // this build resolved rather than from the working directory (#243).
+        skipDirectories: config.skipDirectories,
         packName: name,
         // Which system this pack's documents are stamped for (#48).
         packSystem: system ?? null,
@@ -329,9 +486,60 @@ async function generatePack(
         routingReporter,
     });
     await pack.compile();
+
+    // After the pass, because only now is it known what this pack references.
+    // One folder note materialises in several packs — the items pack and the
+    // journals pack both hold it when both hold something filed in it — and
+    // every copy carries the same `_id`, which is what files a documentation
+    // journal beside the item it describes rather than in a folder that merely
+    // looks alike (#257).
+    writeFolderNoteDocs(
+        referencedFolders,
+        folderNotes,
+        statsForPack(system, config),
+        dest,
+        type,
+        name,
+    );
+
     return { errors: pack.errorCount, compiled: pack.compiledCount };
 }
 
+/**
+ * Write one `Folder` document per referenced folder note into a pack.
+ *
+ * Emitted in address order rather than in the order the pass happened to
+ * reference them, so the same tree compiles to the same bytes on every run.
+ *
+ * @param {Set<import("./folder-notes.mjs").FolderNote>} referenced - The folder
+ *   notes this pack holds something for, ancestors included.
+ * @param {object} folderNotes - The folder-note index.
+ * @param {object} stats - The `_stats` block every emitted document carries.
+ * @param {string} dest - The pack's JSON directory.
+ * @param {string} documentType - The document class the pack holds.
+ * @param {string} packName - Which pack, so each folder takes the parent it has
+ *   *here* rather than the one it has by default.
+ * @returns {void}
+ */
+function writeFolderNoteDocs(referenced, folderNotes, stats, dest, documentType, packName) {
+    if (referenced.size === 0) return;
+    const ordered = [...referenced].sort((a, b) => (a.address < b.address ? -1 : 1));
+    for (const folder of ordered) {
+        const doc = folderDocument(
+            folder,
+            folderNotes.parentOf(folder, packName),
+            documentType,
+            stats,
+        );
+        fs.writeFileSync(
+            path.join(dest, folderFilename(folder.name, folder.id)),
+            JSON.stringify(doc, null, 2),
+            "utf8",
+        );
+    }
+    log.info(`Emitted ${ordered.length} folder-note document(s) to ${dest}`);
+}
+
 /**
  * The passes that compiled nothing when they were expected to compile
  * something — a build failure, not a quiet no-op.
@@ -399,6 +607,33 @@ export async function generatePacksJson({ only, config = loadPackConfig() } = {}
     }
     log.info(`Content tree: ${noteCount} note(s) at ${contentBase}`);
 
+    // One router per configuration, so every pass agrees about where a note
+    // goes, and the first pack of each document type owns the error message for
+    // a note of that type that goes nowhere. Resolved here because the corpus
+    // below is derived against it, and the corpus is what every reader from
+    // this point on reads (#243).
+    const router = routerFor(config);
+
+    // The corpus every pass runs over, and the three whole-tree indexes built
+    // over it, derived **once** for the whole compile (#243). Each is a pure
+    // function of (tree, scope, router), none of which varies between passes —
+    // `router` is one object, handed to all of them — so the passes were
+    // deriving the same answers over and over. Compiling `sohl` read every note
+    // twenty times before this: four per pass, five passes.
+    //
+    // Derived here, before the first reader: the unclaimed-type check below is
+    // one, and a check that walked the tree itself would be answering about a
+    // different corpus from the one the passes then compile.
+    const corpusProblems = [];
+    const corpus = await buildCompileCorpus({
+        contentBase,
+        skipDirectories: config.skipDirectories,
+        router,
+        config,
+        problems: corpusProblems,
+    });
+    for (const problem of corpusProblems) emitDiagnostic(problem);
+
     // A note whose `type:` no configured pack claims compiles into nothing, and
     // used to say nothing (#146) — no pass got far enough to reject it, so the
     // silence had no owner. Asked once, of the whole configuration, because
@@ -406,23 +641,77 @@ export async function generatePacksJson({ only, config = loadPackConfig() } = {}
     // every type a system deliberately does not map, which is exactly the
     // silence #79 requires. Independent of `only`, since it is a fact about the
     // configured pack list rather than about which passes this run executes.
-    const unclaimed = unclaimedNoteFindings(config);
+    const unclaimed = unclaimedNoteFindings(config, undefined, { records: corpus.records });
     for (const finding of unclaimed) emitDiagnostic(finding);
 
     fs.mkdirSync(config.paths.packJson, { recursive: true });
 
     // A companion pack has no pass of its own — naming it selects the pass that
     // writes it, so `compile adventures` is not a silent no-op.
+    //
+    // A **prebuilt** pack has no pass either, and for a plainer reason: its
+    // per-document JSON is checked in. Passed over rather than compiled — which
+    // it could not be before #259, since the only prebuilt pack in the wild
+    // holds Adventures and no compiler was registered for that document type,
+    // so the pack failed the build with "no compiler for document type". Now
+    // one is registered, and running it would wipe `build/packs-json/<name>/`
+    // and write nothing into it — then report the empty pass as an error.
     const packs = config.packs.filter(
         (pack) =>
-            !only ||
-            pack.name === only ||
-            pack.companions.some((companion) => companion.name === only),
+            !pack.prebuilt &&
+            (!only ||
+                pack.name === only ||
+                pack.companions.some((companion) => companion.name === only)),
     );
-    // One router per configuration, so every pass agrees about where a note
-    // goes, and the first pack of each document type owns the error message for
-    // a note of that type that goes nowhere.
-    const router = routerFor(config);
+    // Built once for the whole build, not once per pack: a folder note is one
+    // definition with one address, and every pass resolves against the same
+    // index. A dangling `parent` or a parent cycle is therefore reported once,
+    // as a fact about the tree, rather than once per pass that happened to walk
+    // it (#256).
+    let folderNotes;
+    try {
+        folderNotes = buildFolderNoteIndex(
+            collectFolderNotes(
+                // The corpus this compile derived picks the notes (#243); the
+                // file supplies their frontmatter, and this is one of the few
+                // places where that distinction is load-bearing rather than
+                // incidental.
+                //
+                // `collectFolderNotes` treats `fm.id` as an **authored pin**,
+                // which wins over the id it derives under the folder namespace.
+                // A record's `id` is not that: the index fills it in for every
+                // addressable note (#270), so handing records straight over
+                // would make every folder look pinned and file each one under a
+                // different id than the packs address it by. The index cannot
+                // tell a pin from a derivation, so the note is read — and only
+                // folder notes are, 79 of `sohl`'s 1,685 rather than all of
+                // them.
+                corpus.records
+                    .filter(
+                        (record) =>
+                            isNoteRecord(record) &&
+                            String(record.type ?? "").toLowerCase() === FOLDER_TYPE,
+                    )
+                    .map((record) => {
+                        const absPath = noteFile(contentBase, record);
+                        return { frontmatter: parseMarkdownFile(absPath).frontmatter, absPath };
+                    }),
+                // The package this build resolved, not the ambient accessor:
+                // they are the same value in a real repository and different
+                // ones under `PACKAGE_BUILD_CONFIG`, in a worktree, or in a
+                // test (#243).
+                config.contentPackage,
+            ),
+        );
+    } catch (err) {
+        emitDiagnostic({
+            file: err.absPath ?? contentBase,
+            severity: "error",
+            message: err.message,
+        });
+        return 1;
+    }
+
     const firstOfType = new Map();
     for (const pack of config.packs) {
         if (!firstOfType.has(pack.type)) firstOfType.set(pack.type, pack.name);
@@ -450,7 +739,7 @@ export async function generatePacksJson({ only, config = loadPackConfig() } = {}
         return unsatisfied.length + unclaimed.length;
     }
 
-    let totalErrors = unclaimed.length;
+    let totalErrors = unclaimed.length + corpusProblems.length;
     const passes = [];
     for (const pack of ordered) {
         const { errors, compiled } = await generatePack(
@@ -458,6 +747,8 @@ export async function generatePacksJson({ only, config = loadPackConfig() } = {}
             config,
             router,
             firstOfType.get(pack.type) === pack.name,
+            folderNotes,
+            corpus,
         );
         totalErrors += errors;
         passes.push({