@heroiclands/package-build 17.2.0 → 18.0.0

package/engine/base-compiler.mjs

@@ -66,23 +66,26 @@ import path from "path";
 import log from "loglevel";
 
 import {
-    walkMarkdownTree,
+    parseMarkdownFile,
     makeFilename,
     resolveName,
-    buildContentLinkIndex,
     convertNoteWikilinks,
-    collectContentDocs,
     expandNoteTables,
     statsForPack,
 } from "./helpers.mjs";
+// The record accessors only — see `engine/index-records.mjs` for why they live
+// apart from the index that builds them (#243).
+import { isNoteRecord, noteFile } from "./index-records.mjs";
 import { emitDiagnostic } from "./diagnostics.mjs";
 import { assertNoDeclaredPackage } from "./note-package.mjs";
 import {
     assertNoAliasesField,
     assertNoDraftField,
     assertNoSectionField,
+    assertNoTraitsField,
 } from "./retired-fields.mjs";
 import { assertTypeNotRetired, packForType } from "./ids.mjs";
+import { resolveNoteId } from "./note-ids.mjs";
 import { carriesSystemBlock } from "./system-block.mjs";
 import { checkAuthoredSystemData, checkEmittedSystemData } from "./schema-check.mjs";
 import { locateFrontmatterKey } from "./retired-fields.mjs";
@@ -265,6 +268,8 @@ export class BasePackCompiler {
      * @param {object} options
      * @param {string} options.contentBase - Root of the content tree.
      * @param {string} options.dest - Where this pass writes its JSON.
+     * @param {readonly string[]} options.skipDirectories - Directories the walk
+     *   never descends into. Required: see {@link assertStatedScope}.
      * @param {(path: string|null) => string|null} [options.folderResolver] -
      *   Resolves a `sohl.folder` id against this pack's folder hierarchy.
      * @param {string} [options.packName] - The pack this pass writes.
@@ -277,12 +282,14 @@ export class BasePackCompiler {
     constructor({
         contentBase,
         dest,
+        skipDirectories,
         folderResolver = () => null,
         packName,
         packSystem = null,
         docType,
         router,
         routingReporter = false,
+        corpus,
     } = {}) {
         if (!contentBase) {
             throw new Error(`${this.constructor.name} compiler requires \`contentBase\``);
@@ -290,6 +297,13 @@ export class BasePackCompiler {
         if (!fs.existsSync(contentBase)) {
             throw new Error(`Content tree not found at ${contentBase}`);
         }
+        if (skipDirectories === undefined) {
+            throw new Error(
+                `${this.constructor.name} compiler requires \`skipDirectories\`: ` +
+                    `the walk's scope is stated by whoever builds the pass, not ` +
+                    `resolved from the working directory`,
+            );
+        }
         Object.defineProperty(this, "contentBase", {
             value: contentBase,
             writable: false,
@@ -302,11 +316,24 @@ export class BasePackCompiler {
             value: folderResolver,
             writable: false,
         });
+        // The walk's scope, stated by whoever built this pass rather than
+        // resolved from the working directory (#243). Every walk this compiler
+        // makes — its own, the table corpus, the link index, the SQL tables —
+        // uses this one answer.
+        Object.defineProperty(this, "skipDirectories", {
+            value: skipDirectories,
+            writable: false,
+        });
         this.packName = packName;
         this.packSystem = packSystem;
         this.docType = docType;
         this.router = router;
         this.routingReporter = routingReporter;
+        // The corpus this compile is running over, derived once by
+        // `generatePacksJson` and shared by every pass (#243). A pass that is
+        // handed none derives its own in `prepare`, which is what a consumer
+        // constructing one compiler directly does.
+        this.corpus = corpus;
     }
 
     /**
@@ -357,13 +384,27 @@ export class BasePackCompiler {
      * instances of.
      *
      * @param {object} fm - The note's frontmatter.
-     * @returns {boolean} True when the note may be compiled here.
-     * @throws {Error} When this pack's system is absent from the note. The
-     *   error carries a `position` where the note's own file can be read.
+     * @returns {boolean} True when the note may be compiled here; `false` when
+     *   it belongs to another system's pass of the same document type, which is
+     *   skipped as quietly as any other note this pack does not own.
+     * @throws {Error} When this pack's system is absent from the note and no
+     *   other configured system claims it. The error carries a `position` where
+     *   the note's own file can be read.
      */
     eligibleFor(fm) {
         if (!this.constructor.requiresSystemBlock || !this.packSystem) return true;
         if (carriesSystemBlock(fm, this.packSystem)) return true;
+        // Another system's pack of this document type will claim it (#139). A
+        // note carrying only `hm3:` routes here because this pack is the
+        // *default* of its document type, and defaults are declared per type
+        // rather than per system — but it is not an incomplete note, it is
+        // another pass's. The single-system case is untouched: with no second
+        // system declared there is nothing for this to find, and the error
+        // below still fires.
+        const claimant = (this.router?.systemsOfType?.(this.docType) ?? []).find(
+            (system) => system !== this.packSystem && carriesSystemBlock(fm, system),
+        );
+        if (claimant) return false;
         const label = fm?.name?.full ?? fm?.shortcode ?? fm?.id ?? "this note";
         throw new Error(
             `${label} carries no \`${this.packSystem}:\` block, so it has no ` +
@@ -425,9 +466,35 @@ export class BasePackCompiler {
      * @returns {Promise<void>}
      */
     async prepare() {
+        // The corpus, and the three whole-tree derivations built over it. Every
+        // one of them is a pure function of (tree, scope, router), which do not
+        // vary between the passes of a single compile — so `generatePacksJson`
+        // derives them once and hands them to each pass (#243).
+        //
+        // The measurement that motivated it: compiling `sohl` read every note
+        // **20 times**, four per pass — this link index, the table corpus, the
+        // `sql` scan, and the pass's own walk — across five passes.
+        // Imported here rather than at module scope: deriving the corpus
+        // reaches the pack router and the manifest emitter, which reach this
+        // module, so a static import would close a cycle. `generate.mjs`
+        // normally supplies the corpus and this path never runs.
+        const { buildCompileCorpus } = await import("./compile-corpus.mjs");
+        if (!this.corpus) {
+            this.corpus = await buildCompileCorpus({
+                contentBase: this.contentBase,
+                skipDirectories: this.skipDirectories,
+                router: this.router,
+            });
+            // Derived here, so reported here. A corpus handed in was derived by
+            // `generatePacksJson`, which has already reported its problems
+            // once — and reporting them again in each of five passes would say
+            // the same thing six times.
+            this.reportsCorpusProblems = true;
+        }
         if (this.constructor.convertsWikilinks) {
-            this.linkIndex = buildContentLinkIndex(this.contentBase, this.router);
-            this.contentDocs = collectContentDocs(this.contentBase);
+            this.linkIndex = this.corpus.linkIndex;
+            this.contentDocs = this.corpus.contentDocs;
+            this.sqlTables = this.corpus.sqlTables;
         }
         this.unresolvedLinks = 0;
     }
@@ -452,6 +519,7 @@ export class BasePackCompiler {
             name,
             fm,
             bodyLine,
+            sqlTables: absPath ? this.sqlTables?.get(absPath) : undefined,
         });
         const { markdown, unresolved } = convertNoteWikilinks(tabulated, {
             type: fm.type,
@@ -534,7 +602,15 @@ export class BasePackCompiler {
      */
     reportUndeclaredSystemData(fm, block, documentType, subType) {
         const absPath = this.currentNote?.absPath;
-        const findings = checkAuthoredSystemData(fm, { block, documentType, subType });
+        // Whose schema, where a build has more than one system: this pack's
+        // (#139). `undefined` — a pack that declares no system — keeps the
+        // package-wide answer this always used.
+        const findings = checkAuthoredSystemData(fm, {
+            block,
+            documentType,
+            subType,
+            system: this.packSystem ?? undefined,
+        });
         for (const finding of findings) {
             this.errorCount++;
             const leaf = finding.path.split(".").pop();
@@ -550,7 +626,7 @@ export class BasePackCompiler {
      * The sibling of {@link BasePackCompiler#reportUndeclaredSystemData}, and
      * the half that sees what no declaration states. A compiler writes keys of
      * its own alongside the declared fields — `shortcode`, `actionDefs`,
-     * `notes`, `docHtml`, `archetype` — and neither the field-declaration check
+     * `notes`, `docHtml`, `templatePriority` — and neither the field-declaration check
      * nor the authored-`system` check can see them, so until this nothing
      * compared them at all. Foundry's discard is the same silent one either
      * way.
@@ -575,14 +651,16 @@ export class BasePackCompiler {
      * @param {object} [opts.config] - The resolved build configuration.
      * @returns {number} How many findings were new to this pass.
      */
-    reportEmittedSystemData(system, { fm, block, documentType, subType, type, fields, config }) {
-        const findings = checkEmittedSystemData(system, {
+    reportEmittedSystemData(emitted, { fm, block, documentType, subType, type, fields, config }) {
+        const findings = checkEmittedSystemData(emitted, {
             fm,
             block,
             documentType,
             subType,
             type,
             fields,
+            // See the sibling above: this pack's system, where there is one.
+            system: this.packSystem ?? undefined,
             ...(config ? { config } : {}),
         });
         let added = 0;
@@ -754,9 +832,40 @@ export class BasePackCompiler {
         const label = this.constructor.label;
         const Label = label.charAt(0).toUpperCase() + label.slice(1);
 
-        for (const { frontmatter: fm, body, absPath, bodyLine, bodyColumn } of walkMarkdownTree(
-            this.contentBase,
-        )) {
+        // A note the index could not record is a note this pass declines —
+        // counted and reported exactly as the loop below does for the same
+        // refusal, because it *is* the same refusal: the index defers to
+        // `assertNoDeclaredPackage` for a retired `package:`, which is the
+        // check this loop makes a few lines further down. All that changed is
+        // which pass sees the note first (#243).
+        if (this.reportsCorpusProblems) {
+            for (const problem of this.corpus.problems ?? []) {
+                stats.declined++;
+                this.errorCount++;
+                // Emitted directly rather than through `noteError`, which reads
+                // the file from `currentNote` — the loop has not started, so
+                // there is no current note and the problem carries its own.
+                emitDiagnostic({
+                    file: problem.file,
+                    line: problem.line,
+                    column: problem.column,
+                    severity: "error",
+                    message: problem.message,
+                });
+            }
+        }
+
+        // The corpus, from the index this compile derived once — not a walk of
+        // this pass's own (#243). Each note is then read for its **prose**: the
+        // index carries what is *about* a note and deliberately not its text,
+        // nor the `bodyLine`/`bodyColumn` a diagnostic needs, and this pass has
+        // to have the body anyway. So the read is one this pass was already
+        // making; what it no longer does is decide for itself which files to
+        // make it over.
+        for (const record of this.corpus.records) {
+            if (!isNoteRecord(record)) continue;
+            const absPath = noteFile(this.contentBase, record);
+            const { frontmatter: fm, body, bodyLine, bodyColumn } = parseMarkdownFile(absPath);
             // Which note this pass is on, so anything it calls can report a
             // position without every method having to be handed one (#17).
             this.currentNote = { absPath, bodyLine, bodyColumn };
@@ -789,6 +898,7 @@ export class BasePackCompiler {
                 assertNoDraftField(fm, { absPath });
                 assertNoAliasesField(fm, { absPath });
                 assertNoSectionField(fm, { absPath });
+                assertNoTraitsField(fm, { absPath });
             } catch (err) {
                 stats.declined++;
                 this.errorCount++;
@@ -805,12 +915,29 @@ export class BasePackCompiler {
                 stats.skippedOther++;
                 continue;
             }
+            // The id this note's document is filed under: its authored `id`
+            // if it pins one, otherwise the id derived from its canonical
+            // address (#270). Resolved for every note this pass claims, and
+            // through the one function every other corpus reader calls — the
+            // wikilink index, the content index and the Foundry-address pass
+            // must all compute the id this pass compiles under, and none of
+            // them can see this answer.
+            resolveNoteId(fm);
             if (!fm.id) {
+                // What is left is a note with **no address** — no `type`, or no
+                // `shortcode`. It is not addressable, so there is nothing to
+                // derive from and nothing for a link to point at; the message
+                // names the reason rather than the missing field, because the
+                // field is no longer something an author writes.
                 if (this.constructor.requiresId) {
-                    throw new Error(`${Label} missing id: ${absPath}`);
+                    throw new Error(
+                        `${Label} note has no address, so it has no document id: ` +
+                            `${absPath} — a note is addressed as ` +
+                            `"<type>-<shortcode>" and must declare both`,
+                    );
                 }
                 stats.skippedNoId++;
-                this.noteWarn(`${label} note has no id, skipping`);
+                this.noteWarn(`${label} note has no address to derive an id from, skipping`);
                 continue;
             }
             // Which pack of this type takes it. Applied after the id check —
@@ -834,7 +961,10 @@ export class BasePackCompiler {
             // pass's business — and before `skipNote`, so a pass's own
             // rejection rules never run on a note it may not compile.
             try {
-                this.eligibleFor(fm);
+                if (!this.eligibleFor(fm)) {
+                    stats.skippedOther++;
+                    continue;
+                }
             } catch (err) {
                 stats.declined++;
                 this.errorCount++;

package/engine/bundle-notes.mjs

@@ -0,0 +1,276 @@
+/*
+ * This file is part of the Song of Heroic Lands (SoHL) system for Foundry VTT.
+ * Copyright (c) 2024-2026 Tom Rodriguez ("Toasty") — <toasty@heroiclands.org>
+ *
+ * This work is licensed under the GNU General Public License v3.0 (GPLv3).
+ * You may copy, modify, and distribute it under the terms of that license.
+ *
+ * For full terms, see the LICENSE.md file in the project root or visit:
+ * https://www.gnu.org/licenses/gpl-3.0.html
+ *
+ * SPDX-License-Identifier: GPL-3.0-or-later
+ */
+
+/**
+ * Bundles, as notes — the Foundry `Adventure` a `type: bundle` note compiles
+ * into.
+ *
+ * An `Adventure` is badly named, and the name misled the first design: it is
+ * not a story. `Adventure.contentFields` maps each `SetField` on the schema to
+ * a document class, and importing one partitions its members by whether the
+ * world's collection already holds that `_id`, then creates or updates each.
+ * Afterwards the documents live independently and the Adventure has no further
+ * role. **It is an installer** — a set of document *copies* packaged for
+ * one-shot import (#259).
+ *
+ * That is the whole difference from a folder (#256), which is a live grouping
+ * **by reference** that persists in the pack. A bundle carries copies and
+ * exists to be imported once, so the two are not variations on one idea.
+ *
+ * The note type is `bundle` rather than `adventure` because the format prefers
+ * the domain word wherever Foundry's misleads — a `Scene` is authored as a
+ * `map` — and `collection` was refused because `DocumentCollection` is a real
+ * Foundry class meaning very nearly the opposite. See `docs/content-format.md`
+ * § `type: bundle`.
+ *
+ * This module is the framework-free half: what a bundle note *says*, and how a
+ * set of already-compiled documents becomes an Adventure. The pass that walks
+ * the tree and resolves what one note says about another is
+ * {@link module:engine/bundles}.
+ *
+ * @module
+ */
+
+/**
+ * The note type a bundle is authored as.
+ *
+ * @type {string}
+ */
+export const BUNDLE_TYPE = "bundle";
+
+/**
+ * Foundry's `Adventure.contentFields`, keyed by the document class each holds.
+ *
+ * Restated here rather than derived, because this build never loads Foundry:
+ * the schema is read from `common/documents/adventure.mjs`, where every
+ * `SetField` of an `EmbeddedDataField` is a content field and its name is the
+ * key an importer partitions on. The inverse direction is the one a compiler
+ * wants — it has a document type in hand and needs the field to file it under.
+ *
+ * The full set is listed, not only the five this build compiles, because the
+ * map is a statement about Foundry rather than about this toolchain: a consumer
+ * registering a compiler for `RollTable` needs no edit here.
+ *
+ * @type {Readonly<Record<string, string>>}
+ */
+export const ADVENTURE_CONTENT_FIELD = Object.freeze({
+    Actor: "actors",
+    Combat: "combats",
+    Item: "items",
+    JournalEntry: "journal",
+    Scene: "scenes",
+    RollTable: "tables",
+    Macro: "macros",
+    Cards: "cards",
+    Playlist: "playlists",
+    Folder: "folders",
+});
+
+/**
+ * The `data:` key a bundle lists its members under.
+ *
+ * @type {string}
+ */
+export const CONTENTS_FIELD = "contents";
+
+/**
+ * Strip the LevelDB keys from a document tree.
+ *
+ * An Adventure's members are **inline source data** in a `SetField`, not
+ * sublevel documents, so they carry no `_key`: the compendium CLI's hierarchy
+ * does not recurse into an adventure, and Foundry's schema has no such field to
+ * hold one. A member that kept its key would ship a property the data model
+ * refuses.
+ *
+ * Shared with the scenes pass, which bundles the Adventures that make a pinned
+ * map's ids resolve — one rule about what an Adventure member may carry, stated
+ * once.
+ *
+ * @param {*} value - A document, array, or scalar.
+ * @returns {*} The same shape with every `_key` removed.
+ */
+export function stripAdventureKeys(value) {
+    if (Array.isArray(value)) return value.map(stripAdventureKeys);
+    if (value && typeof value === "object") {
+        return Object.fromEntries(
+            Object.entries(value)
+                .filter(([k]) => k !== "_key")
+                .map(([k, v]) => [k, stripAdventureKeys(v)]),
+        );
+    }
+    return value;
+}
+
+/**
+ * An authored address with any wikilink brackets and label stripped.
+ *
+ * The specification types `contents` as a `WikiLink[]`, and a frontmatter link
+ * is written as a bare address — but `[[address]]` is what an author reaches
+ * for, and it is what a folder note's `parent` already accepts. Accepting both
+ * costs one regex and removes a failure whose message would have to explain the
+ * difference.
+ *
+ * @param {unknown} value - As authored.
+ * @returns {string|null} The bare address, or `null` for a blank entry.
+ */
+export function bareAddress(value) {
+    if (value == null) return null;
+    const text = String(value).trim();
+    if (!text) return null;
+    const unwrapped = text.replace(/^\[\[(.*)\]\]$/s, "$1");
+    // A label is presentation; the address is everything before the pipe.
+    const [target] = unwrapped.split("|");
+    return target.trim() || null;
+}
+
+/**
+ * The addresses a bundle note names, in the order it names them.
+ *
+ * `contents` is a `data:` property, which is where the specification's
+ * `### type: bundle` table puts it — the closed container, so a misspelled
+ * `content` is a finding rather than a silently empty Adventure. It is accepted
+ * at the top level too, exactly as a folder note's `parent` is: an author
+ * following #259's own example rather than the specification should get a
+ * bundle, not a silent default.
+ *
+ * **Order is the author's**, and it is kept: an Adventure's `SetField` attaches
+ * no meaning to member order, but the emitted JSON is compared between runs, so
+ * a stable order is what keeps the same tree compiling to the same bytes.
+ *
+ * @param {object} fm - Parsed frontmatter.
+ * @returns {string[]} The bare addresses. Empty when the note states none.
+ * @throws {Error} When `contents` is neither absent nor a list — a scalar is a
+ *   bundle of one written wrongly, and reading it as such would accept a shape
+ *   the specification does not admit.
+ */
+export function bundleContents(fm) {
+    const data = fm?.data && typeof fm.data === "object" ? fm.data : {};
+    const authored = data[CONTENTS_FIELD] ?? fm?.[CONTENTS_FIELD];
+    if (authored == null) return [];
+    if (!Array.isArray(authored)) {
+        throw new Error(
+            `\`contents\` is a list of addresses, and this note writes a ` +
+                `${typeof authored} — write \`contents:\` as a YAML list, one ` +
+                `address per entry, even where the bundle holds only one`,
+        );
+    }
+    const addresses = [];
+    for (const entry of authored) {
+        const address = bareAddress(entry);
+        if (address) addresses.push(address);
+    }
+    return addresses;
+}
+
+/**
+ * What a pass should do about a member its sources do not hold.
+ *
+ * The whole of the system rule, in one predicate, because it is a rule and not
+ * an implementation detail. An `Adventure` has no `system` field, so a bundle
+ * spanning two systems cannot be one document that knows it spans them: it is
+ * one Adventure per system, and **the pack each is written to is what carries
+ * the system**.
+ *
+ * The constraint is not computed from the member's *type* — that would be wrong
+ * for the types both systems map, which is most of them: a `miscgear` is a SoHL
+ * item **and** an HM3 item, so "which system does this type belong to" has no
+ * single answer. It is read instead from what the pack can actually see. A pack
+ * declaring `system: hm3` reads the HM3 packs and the system-neutral ones, so a
+ * note that publishes no HM3 document is simply not in its sources — which is
+ * the same fact, established where it is already true.
+ *
+ * - **The pack declares a system.** A member it cannot see is another system's,
+ *   and is left **out** — reported, never silently, because an installer that
+ *   quietly ships half its contents is worse than one that fails.
+ * - **The pack declares none.** Nothing is scoped away, so there is no other
+ *   system for the member to have gone to: its absence is a **failure**.
+ *
+ * @param {string|null|undefined} packSystem - The pack's declared `system:`.
+ * @returns {"omit"|"fail"} What to do about a member the sources lack.
+ */
+export function missingMemberVerdict(packSystem) {
+    return packSystem ? "omit" : "fail";
+}
+
+/**
+ * Assemble one `Adventure` from a set of already-compiled documents.
+ *
+ * The members are **copies**, and they arrive compiled: this takes the JSON a
+ * previous pass wrote and files each document under the content field its class
+ * maps to. Nothing is derived from the content tree here, which is what keeps
+ * this half framework-free and testable without one.
+ *
+ * @param {object} params
+ * @param {string} params.id - The Adventure's `_id`.
+ * @param {string} params.name - Its name.
+ * @param {string|null} [params.img] - Its artwork, or `null`.
+ * @param {string} [params.description] - The note's prose, already rendered.
+ * @param {string} [params.caption] - The short caption Foundry shows on the
+ *   import card. Blank unless a caller has one.
+ * @param {string|null} [params.folder] - The folder id it is filed under.
+ * @param {object} [params.flags] - Document flags.
+ * @param {object} params.stats - The `_stats` block to stamp.
+ * @param {ReadonlyArray<{docType: string, document: object}>} params.contents -
+ *   The compiled documents it holds, each with the Foundry class it is.
+ * @returns {object} The Adventure document, keyed for the pack.
+ * @throws {Error} When a member's document class is not one an Adventure can
+ *   hold — a defect in the caller rather than in the note.
+ */
+export function buildAdventure({
+    id,
+    name,
+    img = null,
+    description = "",
+    caption = "",
+    folder = null,
+    flags,
+    stats,
+    contents = [],
+}) {
+    // Every content field, empty, so the emitted document states the whole
+    // schema rather than only the parts this bundle happened to fill. Foundry
+    // defaults an absent `SetField` anyway; writing them all makes two
+    // Adventures diffable against each other.
+    /** @type {Record<string, object[]>} */
+    const held = {};
+    for (const field of Object.values(ADVENTURE_CONTENT_FIELD)) held[field] = [];
+
+    for (const { docType, document } of contents) {
+        const field = ADVENTURE_CONTENT_FIELD[docType];
+        if (!field) {
+            throw new Error(
+                `an Adventure cannot hold ${docType} documents — Foundry's ` +
+                    `\`Adventure.contentFields\` names ` +
+                    `${Object.keys(ADVENTURE_CONTENT_FIELD).join(", ")}`,
+            );
+        }
+        held[field].push(stripAdventureKeys(document));
+    }
+
+    return {
+        name,
+        img,
+        caption,
+        description,
+        ...held,
+        folder,
+        sort: 0,
+        flags: flags || {},
+        _id: id,
+        _stats: stats,
+        // The Foundry *collection* name, not the pack's: a pack directory may
+        // be called anything (#1566), and the key names the collection the
+        // record belongs to inside it.
+        _key: `!adventures!${id}`,
+    };
+}