@heroiclands/package-build 5.0.0 → 6.0.0

package/bin/package-build.mjs

@@ -78,7 +78,7 @@ import yargs from "yargs";
 import { hideBin } from "yargs/helpers";
 
 import { loadPackageBuildConfig } from "../config.mjs";
-import { loadPackConfig } from "../engine/pack-config.mjs";
+import { loadPackConfig, packConfigPath } from "../engine/pack-config.mjs";
 import { cleanBuildArtifacts, stageAssets } from "../stage.mjs";
 import { validateLangSource } from "../lang.mjs";
 import {
@@ -354,6 +354,8 @@ function manifestCommand() {
                 artifact: config.artifact,
                 outDir: path.join(config.rootDir, config.stageDir),
                 flags,
+                // So a `packFolders` finding names the line it is about.
+                configFile: packConfigPath(),
             });
             console.log(
                 `✅ Wrote ${path.relative(config.rootDir, written)} ` +

package/engine/base-compiler.mjs

@@ -34,9 +34,11 @@
  * | {@link BasePackCompiler#finish} | Work that needs every note first. |
  * | {@link BasePackCompiler#reportCompiled} / {@link BasePackCompiler#reportDetail} | The pass's own log lines. |
  *
- * plus two static switches — `requiresId` (a note with no id is fatal, or
- * merely skipped) and `convertsWikilinks` (whether the body reaching
- * `buildEntry` is converted or exactly as authored).
+ * plus three static switches — `requiresId` (a note with no id is fatal, or
+ * merely skipped), `convertsWikilinks` (whether the body reaching
+ * `buildEntry` is converted or exactly as authored) and `readsPackOutputOf`
+ * (the document types whose compiled output this pass reads, which is what the
+ * generator derives the compile order from).
  *
  * `selects` answers *which document type* a pass claims, and it is the same
  * answer for every pack of that type. Which **pack of that type** a claimed
@@ -141,6 +143,29 @@ export class BasePackCompiler {
      */
     static convertsWikilinks = true;
 
+    /**
+     * The document types whose **compiled output** this pass reads.
+     *
+     * Empty for every pass that reads only the content tree. The actors pass
+     * is the exception: a being names its embedded items by
+     * `(type, shortcode)`, and it resolves them against the JSON the item
+     * passes wrote — so an Actor pass must run after every Item pass, and it
+     * says so here.
+     *
+     * The generator derives the compile order from this (#73), so the order
+     * `packs:` declares is presentation only — it is the manifest's `packs`
+     * array as well, and a consumer orders that for a reader. A pass that
+     * reads another's output states the dependency once, in the class that
+     * does the reading, instead of every consuming repository having to know
+     * it when writing its pack list.
+     *
+     * A consumer registering a compiler of its own declares its dependencies
+     * the same way; a type no pack declares is simply not waited for.
+     *
+     * @type {readonly string[]}
+     */
+    static readsPackOutputOf = Object.freeze([]);
+
     /** @type {string} */
     contentBase;
     /** @type {string} */

package/engine/content-links.mjs

@@ -61,6 +61,8 @@ import {
     readCanonicalKey,
 } from "./kb-manifest.mjs";
 import { frontmatterWikilinks, slugify } from "./web-wikilinks.mjs";
+import { homepageAddresses, isHomepage } from "./homepage.mjs";
+import { RETIRED_TYPES } from "./ids.mjs";
 import { parseWikilink, WIKILINK } from "./wikilink-syntax.mjs";
 import { readQualifier } from "./wikilinks.mjs";
 
@@ -305,6 +307,12 @@ export function buildLinkIndex(
         anchors,
         types,
         packages,
+        /**
+         * The one package this tree publishes. Distinct from `packages`, which
+         * is the set an address may name and which a homepage-only tree leaves
+         * this package out of, having no keyed note to put it there.
+         */
+        contentPackage: pkg,
         foreign,
         manifests: manifestsComplete(localPackages, foreign.packages),
         linksOf,
@@ -315,13 +323,223 @@ export function buildLinkIndex(
     };
 }
 
+/**
+ * The site this project publishes on, as a host pattern.
+ *
+ * Hardcoded, as it is in {@link module:engine/homepage} already: every package's
+ * address is `https://www.heroiclands.org/<contentPackage>/`, and the whole
+ * point of the rule below is that an author *should not* be writing that host
+ * into a page. A configurable host would be a second place to write down the
+ * thing being discouraged.
+ *
+ * @type {RegExp}
+ */
+const SITE_HOST = /^(?:[a-z0-9-]+\.)*heroiclands\.org$/i;
+
+/**
+ * How an authored address resolves, or `null` for one nothing here can judge.
+ *
+ * Three shapes reach the site and one does not, and the distinction is the
+ * whole of what is checkable. An address into this site can be reasoned about
+ * from the package roster alone; an address to `github.com`, `kelestia.com` or
+ * `discord.gg` cannot be reasoned about at all without fetching it, and a build
+ * must not depend on a third party being up.
+ *
+ * @param {string} url - The authored address.
+ * @param {ReadonlySet<string>} packages - Package prefixes this build can name.
+ * @returns {{shape: string, segments: string[], prefix: string|null}|null} The
+ *   shape, the path segments, and the package prefix the address starts with.
+ */
+function readAddress(url, packages) {
+    const value = String(url ?? "").trim();
+    if (!value || value.startsWith("#")) return null;
+
+    let segments;
+    let shape;
+    if (/^[a-z][a-z0-9+.-]*:/i.test(value)) {
+        let parsed;
+        try {
+            parsed = new URL(value);
+        } catch {
+            return null;
+        }
+        if (!/^https?:$/.test(parsed.protocol)) return null;
+        if (!SITE_HOST.test(parsed.hostname)) return null;
+        shape = "absolute";
+        segments = parsed.pathname.split("/").filter(Boolean);
+    } else if (value.startsWith("/")) {
+        shape = "rooted";
+        segments = value.split("?")[0].split("#")[0].split("/").filter(Boolean);
+    } else {
+        shape = "relative";
+        segments = value.split("?")[0].split("#")[0].split("/").filter(Boolean);
+    }
+
+    const prefix =
+        shape !== "relative" && packages.has(segments[0]) ? segments[0] : null;
+    return { shape, segments, prefix };
+}
+
+/**
+ * Every defect in the addresses a package homepage carries.
+ *
+ * **Why the homepage needs its own audit at all.** Every other note addresses
+ * the corpus with wikilinks, which {@link auditLinks} resolves. A homepage does
+ * not and cannot: it is published *verbatim* by every publishing mode, including
+ * the homepage-only mode two fan-licensed packages ship under, where the content
+ * tree is never walked and there is no index for a wikilink to resolve against.
+ * So a landing addresses the web the way the web does — markdown links and
+ * `url:` fields — and nothing was looking at those. SoHL's landing pointed at
+ * `kb/creature/` and `kb/character/` from the day those types merged into
+ * `being`: two 404s on the package's front page, through every build.
+ *
+ * **What is checkable, stated plainly.** Only an address into this site is, and
+ * only against facts this build already holds:
+ *
+ * - A **retired content type** in the path. The engine knows what used to exist
+ *   and what replaced it, so this is a fact rather than a guess — and it is
+ *   exactly the SoHL defect.
+ * - A **hardcoded absolute URL** into this package's own prefix, or into one a
+ *   vendored manifest names. Both have a better form to write, which is why they
+ *   are reported; a bare `/<package>/` is left alone, because a package
+ *   homepage is in no manifest and there is nothing better to write.
+ * - A **root-relative `url:`**, which the theme's `relURL` prefixes a second
+ *   time. `href:` means "already resolved, use verbatim", so the same leading
+ *   slash is correct there and is not reported.
+ * - A **wikilink**, which nothing on this page will ever resolve.
+ *
+ * **What is not checkable, and is not attempted.** Whether an external URL
+ * answers — there is no network at build time, and a build must not fail because
+ * a third party is down. And whether a live in-site address names a page that
+ * exists: several of the surfaces a landing routes to are produced by other
+ * tools entirely (generated API documentation, hand-authored Hugo sections), so
+ * this build does not hold the set of published pages and would report a working
+ * link as dead.
+ *
+ * @param {ReturnType<typeof buildLinkIndex>} index - The built index.
+ * @returns {Array<{note: object, field: string, url: string, text: string,
+ *   occurrence: number, message: string}>} One finding per defect, `text` and
+ *   `occurrence` locating it in the note's raw source.
+ */
+export function auditHomepageLinks(index) {
+    const findings = [];
+    const packages = new Set([index.contentPackage, ...index.packages]);
+
+    for (const note of index.notes) {
+        if (!isHomepage(note.fm)) continue;
+
+        // How many times each literal has been seen, so two identical
+        // addresses are located at their own positions.
+        const seen = new Map();
+        const at = (text) => {
+            const occurrence = (seen.get(text) ?? 0) + 1;
+            seen.set(text, occurrence);
+            return occurrence;
+        };
+        const report = (field, url, text, occurrence, message) =>
+            findings.push({ note, field, url, text, occurrence, message });
+
+        for (const [all, rawInner] of matchAllOutsideCode(
+            note.body,
+            new RegExp(WIKILINK.source, "g"),
+        )) {
+            const { target } = parseWikilink(rawInner);
+            report(
+                "body",
+                target,
+                all,
+                at(all),
+                `wikilink ${all} on the package homepage — a homepage is ` +
+                    `published verbatim in every publishing mode, so nothing ` +
+                    `resolves it; write a markdown link, package-relative`,
+            );
+        }
+
+        for (const { field, url, kind } of homepageAddresses(
+            note.fm,
+            note.body,
+        )) {
+            // Counted for every address, checked or not, so the count is
+            // the literal's nth appearance in the file rather than the nth
+            // *finding* about it — two rules can fire on one address.
+            const occurrence = at(url);
+            const address = readAddress(url, packages);
+            if (!address) continue;
+            const { shape, segments, prefix } = address;
+
+            if (shape === "absolute" && prefix) {
+                // A bare `/<package>/` is a package's homepage, which is in no
+                // link manifest and has no relative form from another package.
+                // A finding with no fix is noise.
+                const rest = segments.slice(1).join("/");
+                if (rest) {
+                    report(
+                        field,
+                        url,
+                        url,
+                        occurrence,
+                        prefix === index.contentPackage ?
+                            `hardcoded absolute URL into this package's own ` +
+                                `address — write the package-relative ` +
+                                `"${rest}/", which the landing resolves ` +
+                                `against the site so the page follows the mount`
+                        :   `hardcoded absolute URL into package "${prefix}" ` +
+                                `— resolve it through that package's link ` +
+                                `manifest, whose entries carry the address, so a ` +
+                                `relocation does not leave this page behind`,
+                    );
+                }
+            } else if (shape === "rooted" && kind === "url") {
+                const rest =
+                    prefix ? segments.slice(1).join("/") : segments.join("/");
+                report(
+                    field,
+                    url,
+                    url,
+                    occurrence,
+                    `url "${url}" is root-relative, but a landing's url: is ` +
+                        `resolved against the site — write "${rest}/", or ` +
+                        `href: for an address that is already resolved`,
+                );
+            }
+
+            // The retired-type rule reads the path *inside* the package, so an
+            // address that named one is fixed the same way wherever it was
+            // written.
+            const inPackage = prefix ? segments.slice(1) : segments;
+            for (const [i, segment] of inPackage.entries()) {
+                // `hasOwn`, not a plain lookup: a path segment spelled
+                // `constructor` would otherwise inherit a truthy answer from
+                // `Object.prototype` and be reported as retired.
+                if (!Object.hasOwn(RETIRED_TYPES, segment)) continue;
+                const replacement = RETIRED_TYPES[segment];
+                const fixed = [...inPackage];
+                fixed[i] = replacement;
+                report(
+                    field,
+                    url,
+                    url,
+                    occurrence,
+                    `address "${url}" names content type "${segment}", ` +
+                        `retired in favour of "${replacement}" — both ` +
+                        `compiled to the same document, so the fix is ` +
+                        `mechanical: "${fixed.join("/")}/"`,
+                );
+            }
+        }
+    }
+
+    return findings;
+}
+
 /**
  * Every link in a tree that lands nowhere.
  *
  * @param {ReturnType<typeof buildLinkIndex>} index - The built index.
  * @returns {{deadAnchors: object[], deadAddresses: object[],
- *   frontmatterLinks: object[], usedManifest: Set<string>}} The findings, and
- *   which addresses a foreign manifest answered.
+ *   frontmatterLinks: object[], homepageLinks: object[],
+ *   usedManifest: Set<string>}} The findings, and which addresses a foreign
+ *   manifest answered.
  */
 export function auditLinks(index) {
     const { notes, anchors, linksOf, resolve, manifestHit, isAddress } = index;
@@ -370,6 +588,7 @@ export function auditLinks(index) {
         deadAnchors,
         deadAddresses,
         frontmatterLinks: index.frontmatterLinks,
+        homepageLinks: auditHomepageLinks(index),
         usedManifest,
     };
 }

package/engine/content-lint.mjs

@@ -164,15 +164,24 @@ export function lintContentTree(contentBase, { skipDirectories } = {}) {
     // "Every one of nothing is unique" is a vacuous pass, and it is exactly
     // what a tree that failed to check out produces — so the lint would go
     // green on the one state it most needs to catch.
-    if (byKey.size === 0) {
+    //
+    // The state that catches is an **empty walk**, not an empty key set (#77).
+    // A note may be keyless by design: a homepage carries no `shortcode`
+    // because it is addressed by the package rather than by a slug, so a
+    // package in `publish.site: homepage` mode has a content tree that is
+    // populated, correct, and permanently unkeyed. Reporting that as a missing
+    // checkout trains its author to stop reading the output — the one thing
+    // this guard needs them to do. A tree holding notes is therefore a tree;
+    // only a tree holding none is the absent one.
+    if (notes.length === 0) {
         findings.push({
             file: path.relative(process.cwd(), contentBase) || contentBase,
             severity: "error",
             message:
-                "holds no keyed content, so every rule here is vacuous — " +
+                "holds no content notes, so every rule here is vacuous — " +
                 "check that the content tree is present and that this is its root",
         });
-        return { findings, notes: notes.length, keys: 0 };
+        return { findings, notes: 0, keys: 0 };
     }
 
     for (const [key, files] of byKey) {

package/engine/diagnostics.mjs

@@ -44,6 +44,7 @@
  */
 
 import path from "node:path";
+import YAML, { LineCounter } from "yaml";
 
 /**
  * The `file:line:column` locator, with whatever is known.
@@ -268,3 +269,48 @@ export function positionOfLiteral(text, needle, occurrence = 1) {
         column: at - before.lastIndexOf("\n"),
     };
 }
+
+/**
+ * Where a **key or value in a YAML document** sits, addressed by its path.
+ *
+ * {@link positionOfLiteral} is the plain search, and it is the wrong tool for a
+ * configuration file: a pack name like `items` or `actors` appears in the
+ * `packs:` block, in a folder's list, in a path, and often in prose, so the
+ * first occurrence is routinely not the one the finding is about — which is the
+ * one thing the located form exists to prevent.
+ *
+ * A path resolves the exact node instead. The document is re-parsed here rather
+ * than threaded from the loader because the loader returns plain data: `yaml`
+ * discards ranges once a document is materialised, and carrying a parallel
+ * position tree through configuration resolution would be a second
+ * representation of the same file to keep in step.
+ *
+ * Every failure — unparseable text, an `.mjs` configuration, a path that
+ * resolves to nothing — yields `{}`, so a caller spreads the result and the
+ * position is dropped rather than guessed.
+ *
+ * @param {string} text - The document's contents.
+ * @param {ReadonlyArray<string|number>} keyPath - Path to the node: map keys as
+ *   strings, sequence entries as numbers.
+ * @returns {{line?: number, column?: number}} Spreadable position fields.
+ */
+export function positionOfYamlPath(text, keyPath) {
+    if (typeof text !== "string" || !text) return {};
+    if (!Array.isArray(keyPath) || keyPath.length === 0) return {};
+
+    let node;
+    const counter = new LineCounter();
+    try {
+        const doc = YAML.parseDocument(text, { lineCounter: counter });
+        // `keepScalar` returns the Scalar node rather than its value, which is
+        // the only form carrying a range.
+        node = doc.getIn(keyPath, true);
+    } catch {
+        return {};
+    }
+
+    const start = node?.range?.[0];
+    if (!Number.isFinite(start)) return {};
+    const { line, col } = counter.linePos(start);
+    return { line, column: col };
+}

package/engine/generate.mjs

@@ -23,10 +23,15 @@
  * Each `*` compiler walks the whole content tree and selects its own entries by
  * the note's `type` — every note in the tree belongs to this repository's
  * `contentPackage` (#56) — so routing is directory-agnostic: a file lands in a
- * pack because of its `type`, not its location. Which packs exist, in what order, and which folder
- * hierarchy each one loads are all declared in `package-build.config.yaml`;
- * folder files live under the content root and are referenced from entry
- * frontmatter via `sohl.folder: <id>`.
+ * pack because of its `type`, not its location. Which packs exist and which
+ * folder hierarchy each one loads are declared in
+ * `package-build.config.yaml`; folder files live under the content root and are
+ * referenced from entry frontmatter via `sohl.folder: <id>`.
+ *
+ * **The order the passes run in is derived, not declared** — see
+ * {@link orderPassesByDependency}. The declared list is the manifest's `packs`
+ * array as well, so it is ordered for a reader; a pass that reads another's
+ * output states that on its compiler and is scheduled after it (#73).
  *
  * This replaces the retired `packs:export` (vault → committed `_source/`); the
  * HeroicLands vault is no longer a build input for SoHL content.
@@ -50,6 +55,7 @@ import {
     writeFolderDocs,
 } from "./helpers.mjs";
 import { countContentNotes } from "./content-tree.mjs";
+import { emitDiagnostic } from "./diagnostics.mjs";
 import { loadPackConfig } from "./pack-config.mjs";
 import { routerFor } from "./pack-router.mjs";
 
@@ -89,8 +95,9 @@ export const packJsonDir = (name, config = loadPackConfig()) =>
  * and an actor's embedded items may be sourced from any of them. Finding one
  * pack and stopping is how embedded-item resolution would silently miss every
  * item that landed in another. Returned in configured order, which is also the
- * order they compile in, so a pack later in the list cannot be read before it
- * is written.
+ * order they compile in — {@link orderPassesByDependency} keeps the declared
+ * order among packs of one type — and every one of them is written before the
+ * actors pass that reads them.
  *
  * @param {object} [config] - The resolved build configuration. Defaults to this
  *   repository's.
@@ -104,6 +111,126 @@ export function itemPackJsonDirs(config = loadPackConfig()) {
         .map((pack) => packJsonDir(pack.name, config));
 }
 
+/**
+ * The document types whose compiled output a pass of this type reads.
+ *
+ * Asked of the compiler rather than held in a table here, so the dependency
+ * lives in the class that does the reading and a consumer's own compiler can
+ * declare its own. A type no compiler is registered for waits on nothing —
+ * {@link generatePack} reports it as an unknown type, which is the better
+ * message.
+ *
+ * @param {string} type - A pack's document type.
+ * @returns {readonly string[]} The types it reads the output of.
+ */
+function readsOutputOf(type) {
+    return COMPILERS[type]?.readsPackOutputOf ?? [];
+}
+
+/**
+ * The passes to run, ordered so that each one follows the output it reads.
+ *
+ * **Declaration order is presentation, not compile order (#73).** The same
+ * `packs:` list is the manifest's `packs` array, which a consumer orders for a
+ * reader browsing compendiums; the actors pass, meanwhile, resolves each
+ * being's embedded items against the item passes' *output*. Making one list
+ * satisfy both meant an Actor pack declared first compiled only where a
+ * previous run had already left `build/packs-json` populated — green on a warm
+ * tree, exit 1 on every fresh checkout and every CI runner, and `build/` is
+ * gitignored so that is the state CI always starts from.
+ *
+ * **The reordering is the smallest one that works.** Each step takes the
+ * *earliest declared* pass whose dependencies are all already emitted, so a
+ * list that was already in a workable order comes back untouched, and one that
+ * was not moves exactly the passes that had to move. A dependency is satisfied
+ * only when **every** pack of that type has run: a being addresses an item by
+ * `(type, shortcode)` without knowing which Item pack ships it, so waiting for
+ * one of several would resolve some beings and silently fail others.
+ *
+ * A dependency on a type this configuration declares no pack of is not waited
+ * for. A package may ship an Actor pack and no Item pack; the pass that needs
+ * one refuses on its own, with a message about items rather than about order.
+ *
+ * @param {readonly object[]} packs - The passes to be run, as declared.
+ * @returns {object[]} A new list, in compile order. The input is untouched.
+ * @throws {Error} If the passes read each other's output in a cycle, which no
+ *   order can satisfy. Only reachable from a mis-declared compiler, so it names
+ *   the passes rather than blaming the pack list.
+ */
+export function orderPassesByDependency(packs) {
+    const declared = new Set(packs.map((pack) => pack.type));
+    const remaining = [...packs];
+    const emitted = new Set();
+    /** @type {object[]} */
+    const ordered = [];
+
+    /** Whether every pack this pass reads the output of has already run. */
+    const ready = (pack) =>
+        readsOutputOf(pack.type).every(
+            (dependency) =>
+                !declared.has(dependency) ||
+                packs.every(
+                    (other) =>
+                        other.type !== dependency || emitted.has(other.name),
+                ),
+        );
+
+    while (remaining.length) {
+        const next = remaining.findIndex(ready);
+        if (next === -1) {
+            throw new Error(
+                `package-build: the configured passes read each other's ` +
+                    `output in a cycle (` +
+                    `${remaining.map((pack) => `${pack.name} (${pack.type})`).join(", ")}` +
+                    `); no compile order can satisfy that.`,
+            );
+        }
+        const [pack] = remaining.splice(next, 1);
+        emitted.add(pack.name);
+        ordered.push(pack);
+    }
+    return ordered;
+}
+
+/**
+ * The dependencies this run cannot satisfy by ordering, because the pass that
+ * would produce them is not in it.
+ *
+ * Ordering answers the whole-package build; a run restricted to one pack
+ * (`content-build package compile <name>`) cannot conjure the passes it left
+ * out. Where their output is already on disk from an earlier run that is fine
+ * — it is how compiling one pack at a time is meant to work — so this reports
+ * only what is genuinely absent, and names the pack that would write it rather
+ * than the directory that is missing.
+ *
+ * @param {readonly object[]} running - The passes this run will execute.
+ * @param {object} config - The resolved build configuration.
+ * @returns {string[]} One message per unsatisfiable dependency.
+ */
+export function unsatisfiedPassDependencies(running, config) {
+    const included = new Set(running.map((pack) => pack.name));
+    /** @type {string[]} */
+    const messages = [];
+    for (const pack of running) {
+        for (const dependency of readsOutputOf(pack.type)) {
+            for (const producer of config.packs) {
+                if (producer.type !== dependency) continue;
+                if (included.has(producer.name)) continue;
+                const dir = packJsonDir(producer.name, config);
+                if (fs.existsSync(dir)) continue;
+                messages.push(
+                    `pack "${pack.name}" (${pack.type}) reads the compiled ` +
+                        `output of the ${producer.type} pack ` +
+                        `"${producer.name}", which this run does not compile ` +
+                        `and which ${dir} does not hold — compile the whole ` +
+                        `package, or compile "${producer.name}" first`,
+                );
+            }
+        }
+    }
+    return messages;
+}
+
 /**
  * Generate the per-entry JSON for one pack into `build/packs-json/<name>/`.
  *
@@ -280,9 +407,31 @@ export async function generatePacksJson({
         if (!firstOfType.has(pack.type)) firstOfType.set(pack.type, pack.name);
     }
 
+    // Compile order is derived from what each pass reads, not from the order
+    // `packs:` declares — that list is also the manifest's, which a consumer
+    // orders for a reader (#73).
+    const ordered = orderPassesByDependency(packs);
+    if (ordered.some((pack, index) => pack !== packs[index])) {
+        log.info(
+            `Pass order: ${ordered.map((pack) => pack.name).join(", ")} — a ` +
+                `pass that reads another's output compiles after it, whatever ` +
+                `order \`packs:\` declares.`,
+        );
+    }
+
+    // What ordering cannot reach: a run restricted to one pack, whose
+    // dependencies are simply not in it and not on disk either.
+    const unsatisfied = unsatisfiedPassDependencies(ordered, config);
+    if (unsatisfied.length) {
+        for (const message of unsatisfied) {
+            emitDiagnostic({ severity: "error", message });
+        }
+        return unsatisfied.length;
+    }
+
     let totalErrors = 0;
     const passes = [];
-    for (const pack of packs) {
+    for (const pack of ordered) {
         const { errors, compiled } = await generatePack(
             pack,
             config,