@heroiclands/package-build 6.0.0 → 7.0.0

package/engine/address-diff.mjs

@@ -0,0 +1,290 @@
+/*
+ * This file is part of the Song of Heroic Lands (SoHL) system for Foundry VTT.
+ * Copyright (c) 2024-2026 Tom Rodriguez ("Toasty") — <toasty@heroiclands.org>
+ *
+ * This work is licensed under the GNU General Public License v3.0 (GPLv3).
+ * You may copy, modify, and distribute it under the terms of that license.
+ *
+ * For full terms, see the LICENSE.md file in the project root or visit:
+ * https://www.gnu.org/licenses/gpl-3.0.html
+ *
+ * SPDX-License-Identifier: GPL-3.0-or-later
+ */
+
+/**
+ * Diffing a package's published item addresses against a released one (#66).
+ *
+ * A package's `(type, shortcode)` addresses are a **published interface**.
+ * Every satellite that declares `itemCatalog: true` assembles its beings out of
+ * them — `attribute:str`, `skill:awar`, `weapongear:Tabri` — resolving each one
+ * against the Item packs of the release its `compatibility.verified` pins. So
+ * renaming a shortcode is a breaking change to something other repositories
+ * consume, and until this module there was nothing that noticed: the check that
+ * got made was a repository-local grep, which cannot see the other
+ * repositories and reports the reassuring answer.
+ *
+ * `sohl` renamed one weapon's shortcode from `Tabri` to `Taburi` two days after
+ * the `v0.8.2` tag, on the stated ground that "nothing referenced the old
+ * value, so the rename is self-contained". True of that repository. Both
+ * satellites pin `v0.8.2` and address `weapongear:Tabri` on their copy of the
+ * same character — five lookups that resolve today and fail the moment either
+ * pin moves, with an error reading like a missing item.
+ *
+ * **The comparison is release-to-release, in the repository doing the
+ * renaming.** The alternative — checking a consumer's addresses against its
+ * pinned release — already exists and already fails the build (`no predefined
+ * item for "weapongear:Taburi"`); what it lacks is an explanation, and it
+ * cannot honestly produce one, because at the point of the miss all it holds is
+ * the address string. It has no document id and no name to match a candidate
+ * against, so any successor it named would be a guess at a similar-looking
+ * string. Here both sides are whole documents, so the question is decidable.
+ *
+ * **A rename is told from a removal by the document id, and that is an identity
+ * match rather than an inference.** A note authors its `_id` in frontmatter; it
+ * is not derived from the shortcode, and the `Tabri` → `Taburi` commit changed
+ * the shortcode alone. So an address that disappeared while its document is
+ * still published elsewhere *is* a rename — not "probably" one. When the id is
+ * published under no address at all, that is all this can say: **withdrawn**,
+ * with no successor named. A split, a deletion and a merge are indistinguish-
+ * able from one another at that point, and inventing a "did you mean" from
+ * string similarity would be worse than saying nothing, because a wrong one
+ * sends the reader to the wrong fix.
+ *
+ * **Severity is decided per case.** A withdrawal is legitimate — content is
+ * allowed to be retired — so it is reported and does not fail a build. A rename
+ * is equally legitimate as a decision (#1397's charset rule forces some), which
+ * is why it does not fail one either; what it must not do is happen in silence.
+ * A caller that wants a gate passes `error` and treats any finding as one.
+ *
+ * Item packs only, because that is the address space consumers resolve
+ * against: {@link foreignItemCatalogDirs} extracts nothing else, and a being's
+ * embedded items are the only cross-package resolution by `(type, shortcode)`.
+ *
+ * @module
+ */
+
+import fs from "node:fs";
+import path from "node:path";
+
+import { formatDiagnostic, positionInFrontmatter } from "./diagnostics.mjs";
+import { positionOfLiteral } from "./diagnostics.mjs";
+import { walkMarkdownTree } from "./helpers.mjs";
+
+/**
+ * The address space a set of compiled Item pack directories publishes.
+ *
+ * The directories are read as one space for the same reason the actors pass
+ * reads them as one: a being names an item by `(type, shortcode)` and never by
+ * the pack it happens to ship in. Both sides of a diff are built by this one
+ * function, so a released catalogue extracted by `deps fetch` and a freshly
+ * compiled pack are indexed identically and a difference between them is a real
+ * one rather than an artefact of two readers.
+ *
+ * A missing directory throws rather than reading as an empty space: an empty
+ * baseline would report every address in the package as withdrawn, and an empty
+ * current side would report every address as gone — the loudest possible
+ * output from the quietest possible mistake.
+ *
+ * @param {readonly string[]} dirs - Directories of item JSON.
+ * @returns {Map<string, {id: string, name: string, type: string, shortcode: string, file: string}>}
+ *   Every item, keyed `type:shortcode`.
+ */
+export function readItemAddresses(dirs) {
+    const space = new Map();
+    for (const dir of dirs) {
+        if (!fs.existsSync(dir)) {
+            throw new Error(
+                `Item source directory ${dir} does not exist — an address ` +
+                    `diff reads compiled Item pack output, so those packs ` +
+                    `must be compiled (or the catalogue fetched) first`,
+            );
+        }
+        for (const name of fs.readdirSync(dir)) {
+            if (!name.endsWith(".json")) continue;
+            if (name.startsWith("folder_")) continue;
+            const file = path.join(dir, name);
+            let doc;
+            try {
+                doc = JSON.parse(fs.readFileSync(file, "utf8"));
+            } catch {
+                // Unparseable output is the compile's problem to report, not
+                // this pass's; skipping it here loses one address rather than
+                // failing a diff that has nothing to do with it.
+                continue;
+            }
+            const shortcode = doc?.system?.shortcode;
+            if (!doc?.type || !shortcode || !doc?._id) continue;
+            space.set(`${doc.type}:${shortcode}`, {
+                id: doc._id,
+                name: doc.name ?? "",
+                type: doc.type,
+                shortcode,
+                file,
+            });
+        }
+    }
+    return space;
+}
+
+/**
+ * Every address the baseline published that this build does not.
+ *
+ * An address that merely *arrived* is not a finding: adding one breaks nobody.
+ * The arrivals are read only to answer the one question that matters about a
+ * departure — is the document still here under another name?
+ *
+ * @param {Map<string, object>} baseline - The released address space.
+ * @param {Map<string, object>} current - This build's address space.
+ * @param {object} opts
+ * @param {string} opts.baseline - What the baseline is, for the message —
+ *   conventionally `<package>@<version>`.
+ * @returns {Array<object>} One finding per departed address, in address order
+ *   so two runs read the same. `kind` is `"renamed"` (with `to`) or
+ *   `"withdrawn"`.
+ */
+export function diffItemAddresses(baseline, current, { baseline: label }) {
+    // A baseline that yields no address at all cannot produce a finding, so it
+    // reports a clean result for every possible input — the one failure a check
+    // like this can never catch, and the same one `foreign-manifests.mjs` exists
+    // to stop. It is a real state, not a hypothetical: `sohl-kethira-basic@0.5.3`
+    // shipped 307 items carrying no `system.shortcode` between them.
+    if (!baseline.size) {
+        throw new Error(
+            `${label} publishes no addressable item — no document in its Item ` +
+                `packs carries a \`system.shortcode\`. A diff against it can ` +
+                `only report that nothing changed, whatever this build does, ` +
+                `so it is refused rather than passed`,
+        );
+    }
+    // Where each still-published document lives now. Built once: a rename is
+    // decided by identity, so this is the whole evidence base.
+    const currentById = new Map();
+    for (const [address, entry] of current) {
+        if (!currentById.has(entry.id)) currentById.set(entry.id, address);
+    }
+
+    const findings = [];
+    for (const [address, entry] of baseline) {
+        if (current.has(address)) continue;
+        const to = currentById.get(entry.id);
+        findings.push({
+            kind: to ? "renamed" : "withdrawn",
+            address,
+            ...(to ? { to } : {}),
+            id: entry.id,
+            name: entry.name,
+            shortcode: entry.shortcode,
+            baselineFile: entry.file,
+            baseline: label,
+        });
+    }
+    findings.sort((a, b) => (a.address < b.address ? -1 : 1));
+    return findings;
+}
+
+/**
+ * Every content note in a tree, indexed by the document id it authors.
+ *
+ * The address space is read from compiled output because that is what actually
+ * ships; the tree is read only to place a finding somewhere a reader can open
+ * and fix it. Each source answers the question it is good at, and the id is the
+ * exact key that joins them.
+ *
+ * @param {string} contentBase - Root of the content tree.
+ * @param {object} [opts]
+ * @param {readonly string[]} [opts.skipDirectories] - Passed to the walk.
+ * @returns {Map<string, string>} Document id → the note's absolute path.
+ */
+export function noteFilesById(contentBase, { skipDirectories } = {}) {
+    const byId = new Map();
+    const walkOpts = skipDirectories ? { skipDirectories } : undefined;
+    for (const { frontmatter: fm, absPath } of walkMarkdownTree(
+        contentBase,
+        walkOpts,
+    )) {
+        if (fm?.id && !byId.has(fm.id)) byId.set(fm.id, absPath);
+    }
+    return byId;
+}
+
+/**
+ * Where to send the reader for one finding.
+ *
+ * A rename is fixed in the note that made it, so a finding whose id is still in
+ * this tree is reported at that note's `shortcode:` line — the line the author
+ * just edited. A withdrawal has no such note by definition, so it degrades to
+ * the baseline document, which is the only artefact left that records the
+ * address existing. When neither is readable the position is **dropped**, never
+ * defaulted to `1:1`.
+ *
+ * @param {object} finding - One finding from {@link diffItemAddresses}.
+ * @param {Map<string, string>} noteFiles - From {@link noteFilesById}.
+ * @returns {{file?: string, line?: number, column?: number}} Spreadable
+ *   position fields for {@link formatDiagnostic}.
+ */
+export function locateAddressFinding(finding, noteFiles) {
+    const note = noteFiles?.get(finding.id);
+    if (note) {
+        try {
+            const raw = fs.readFileSync(note, "utf8");
+            return { file: note, ...positionInFrontmatter(raw, "shortcode") };
+        } catch {
+            return { file: note };
+        }
+    }
+    if (!finding.baselineFile) return {};
+    try {
+        const raw = fs.readFileSync(finding.baselineFile, "utf8");
+        return {
+            file: finding.baselineFile,
+            ...positionOfLiteral(raw, `"${finding.shortcode}"`),
+        };
+    } catch {
+        return { file: finding.baselineFile };
+    }
+}
+
+/**
+ * What one finding says, without a locator or a severity.
+ *
+ * The rename message names the identity it matched on, because that is what
+ * separates this from a spelling suggestion: the reader can check the id in
+ * both artefacts. The withdrawal message names no successor, because none is
+ * known — and says so, rather than leaving the reader to wonder whether one was
+ * looked for.
+ *
+ * @param {object} finding - One finding from {@link diffItemAddresses}.
+ * @returns {string} The message.
+ */
+export function addressFindingMessage(finding) {
+    if (finding.kind === "renamed") {
+        return (
+            `since ${finding.baseline}, ${finding.address} is no longer ` +
+            `published; the same document (${finding.id}) is now published ` +
+            `as ${finding.to}. Every package that resolves ${finding.address} ` +
+            `breaks when it moves past ${finding.baseline}`
+        );
+    }
+    return (
+        `since ${finding.baseline}, ${finding.address} is no longer ` +
+        `published, and its document (${finding.id}) is published under no ` +
+        `other address`
+    );
+}
+
+/**
+ * One finding, in the standard `file:line:column: severity: message` form.
+ *
+ * @param {object} finding - One finding from {@link diffItemAddresses}.
+ * @param {{file?: string, line?: number, column?: number}} at - From
+ *   {@link locateAddressFinding}.
+ * @param {"warning"|"error"} [severity] - `error` when the caller is gating.
+ * @returns {string} The formatted diagnostic, path first on the line.
+ */
+export function formatAddressFinding(finding, at, severity = "warning") {
+    return formatDiagnostic({
+        ...at,
+        severity,
+        message: addressFindingMessage(finding),
+    });
+}

package/engine/base-compiler.mjs

@@ -73,6 +73,7 @@ import {
     convertNoteWikilinks,
     collectContentDocs,
     expandNoteTables,
+    statsForPack,
 } from "./helpers.mjs";
 import { emitDiagnostic } from "./diagnostics.mjs";
 import { assertNoDeclaredPackage } from "./note-package.mjs";
@@ -237,6 +238,7 @@ export class BasePackCompiler {
         dest,
         folderResolver = () => null,
         packName,
+        packSystem = null,
         docType,
         router,
         routingReporter = false,
@@ -262,11 +264,34 @@ export class BasePackCompiler {
             writable: false,
         });
         this.packName = packName;
+        this.packSystem = packSystem;
         this.docType = docType;
         this.router = router;
         this.routingReporter = routingReporter;
     }
 
+    /**
+     * The `_stats` block every entry this pass emits is stamped with (#48).
+     *
+     * Per pack rather than per package, because a module may ship the same
+     * content for two systems — `harn-ensemble` has an `actors-hm3` pack and an
+     * `actors-sohl` pack — and those documents were built against different
+     * system versions. A single global block stamped both identically.
+     *
+     * Memoised on the instance: one pass, one pack, one system, so the block is
+     * constant for the life of the compiler. The previous module-level memo
+     * could not be, because it was shared across passes for different packs.
+     *
+     * @returns {object} The block, built once per compiler.
+     */
+    get stats() {
+        this.#stats ??= statsForPack(this.packSystem);
+        return this.#stats;
+    }
+
+    /** @type {object|undefined} */
+    #stats;
+
     /**
      * Whether this pass's pack is the one a claimed note belongs in.
      *

package/engine/content-links.mjs

@@ -58,6 +58,7 @@ import {
     canonicalKey,
     loadForeignManifests,
     manifestsComplete,
+    PACKAGE_BASE,
     readCanonicalKey,
 } from "./kb-manifest.mjs";
 import { frontmatterWikilinks, slugify } from "./web-wikilinks.mjs";
@@ -336,6 +337,74 @@ export function buildLinkIndex(
  */
 const SITE_HOST = /^(?:[a-z0-9-]+\.)*heroiclands\.org$/i;
 
+/**
+ * Every package landing this build can name, as `package` → base (#87).
+ *
+ * **A landing needs no manifest, and that is what makes it work.** The link
+ * manifest indexes content notes, and a homepage is deliberately not one — it
+ * compiles to no document and is entered in no manifest. The reading that
+ * follows from this, and that left a hardcoded URL as the only authored form,
+ * is that a landing therefore cannot be addressed. It does not follow: a
+ * landing's address is not a *note's* address but the **package's**, and
+ * {@link PACKAGE_BASE} already records where each package is served. That is a
+ * frozen constant vendored into every repository, so consulting it walks no
+ * tree, reads no manifest and builds no index — which is precisely why the
+ * mechanism survives `homepage` mode, where the licensing fence means none of
+ * those exist.
+ *
+ * The roster is consulted **for landings only**. Widening the package set the
+ * other rules read would make them offer manifest-based advice about packages
+ * no manifest is vendored for.
+ *
+ * @param {string} ownPackage - The package this build publishes.
+ * @param {Iterable<string>} manifestPackages - Packages a vendored manifest
+ *   names, which are addressable whether or not the roster lists them.
+ * @returns {Map<string, string>} Package to base, each base slash-terminated.
+ */
+function landingBases(ownPackage, manifestPackages) {
+    const bases = new Map();
+    // Convention first, roster second, so a package the roster relocates is
+    // recorded at the relocated base rather than the default one.
+    for (const pkg of [ownPackage, ...manifestPackages]) {
+        if (pkg) bases.set(pkg, `/${pkg}/`);
+    }
+    for (const [pkg, base] of Object.entries(PACKAGE_BASE)) {
+        if (typeof base === "string" && base.endsWith("/")) {
+            bases.set(pkg, base);
+        }
+    }
+    return bases;
+}
+
+/**
+ * The package whose landing an address names, or `null`.
+ *
+ * Matches the whole path, not a prefix: `/sohl/` is the landing, `/sohl/kb/`
+ * is a page inside the package and belongs to the manifest rules instead.
+ *
+ * @param {string} url - The authored address.
+ * @param {Map<string, string>} bases - From {@link landingBases}.
+ * @returns {{pkg: string, base: string}|null} The package and its base.
+ */
+function landingTarget(url, bases) {
+    const value = String(url ?? "").trim();
+    if (!value || !/^[a-z][a-z0-9+.-]*:/i.test(value)) return null;
+    let parsed;
+    try {
+        parsed = new URL(value);
+    } catch {
+        return null;
+    }
+    if (!/^https?:$/.test(parsed.protocol)) return null;
+    if (!SITE_HOST.test(parsed.hostname)) return null;
+    const pathname =
+        parsed.pathname.endsWith("/") ? parsed.pathname : `${parsed.pathname}/`;
+    for (const [pkg, base] of bases) {
+        if (pathname === base) return { pkg, base };
+    }
+    return null;
+}
+
 /**
  * How an authored address resolves, or `null` for one nothing here can judge.
  *
@@ -400,9 +469,17 @@ function readAddress(url, packages) {
  *   and what replaced it, so this is a fact rather than a guess — and it is
  *   exactly the SoHL defect.
  * - A **hardcoded absolute URL** into this package's own prefix, or into one a
- *   vendored manifest names. Both have a better form to write, which is why they
- *   are reported; a bare `/<package>/` is left alone, because a package
- *   homepage is in no manifest and there is nothing better to write.
+ *   vendored manifest names. Every one of them has a better form to write, which
+ *   is why every one is reported — including a bare `/<package>/`, which names
+ *   another package's landing (#87).
+ *
+ *   That last case was exempt until the better form was identified, on the
+ *   reasoning that a landing is in no link manifest so nothing could resolve it.
+ *   True, and beside the point: it does not need resolving. A landing's address
+ *   *is* its package prefix, so `/<package>/` is the absolute URL with the host
+ *   struck off — host-free, emitted verbatim, and needing no index, which is
+ *   what lets it hold in homepage-only mode where the tree is never walked. The
+ *   form was already accepted here; nothing had ever named it as the one to use.
  * - A **root-relative `url:`**, which the theme's `relURL` prefixes a second
  *   time. `href:` means "already resolved, use verbatim", so the same leading
  *   slash is correct there and is not reported.
@@ -424,6 +501,7 @@ function readAddress(url, packages) {
 export function auditHomepageLinks(index) {
     const findings = [];
     const packages = new Set([index.contentPackage, ...index.packages]);
+    const bases = landingBases(index.contentPackage, index.packages);
 
     for (const note of index.notes) {
         if (!isHomepage(note.fm)) continue;
@@ -467,28 +545,43 @@ export function auditHomepageLinks(index) {
             if (!address) continue;
             const { shape, segments, prefix } = address;
 
-            if (shape === "absolute" && prefix) {
-                // A bare `/<package>/` is a package's homepage, which is in no
-                // link manifest and has no relative form from another package.
-                // A finding with no fix is noise.
+            // Landings first, and by the roster rather than by the manifest
+            // package set: a landing is addressable in a repository that
+            // vendors no manifest at all, which is the case the fence creates
+            // and the case this rule exists for (#87).
+            const landing = landingTarget(url, bases);
+            if (landing) {
+                report(
+                    field,
+                    url,
+                    url,
+                    occurrence,
+                    `hardcoded absolute URL to ` +
+                        (landing.pkg === index.contentPackage ?
+                            `this package's own landing`
+                        :   `package "${landing.pkg}"'s landing`) +
+                        ` — write "${landing.base}", which names no host, is ` +
+                        `emitted verbatim, and resolves through the package ` +
+                        `roster rather than through an index, so it holds ` +
+                        `where no content tree is walked`,
+                );
+            } else if (shape === "absolute" && prefix) {
                 const rest = segments.slice(1).join("/");
-                if (rest) {
-                    report(
-                        field,
-                        url,
-                        url,
-                        occurrence,
-                        prefix === index.contentPackage ?
-                            `hardcoded absolute URL into this package's own ` +
-                                `address — write the package-relative ` +
-                                `"${rest}/", which the landing resolves ` +
-                                `against the site so the page follows the mount`
-                        :   `hardcoded absolute URL into package "${prefix}" ` +
-                                `— resolve it through that package's link ` +
-                                `manifest, whose entries carry the address, so a ` +
-                                `relocation does not leave this page behind`,
-                    );
-                }
+                report(
+                    field,
+                    url,
+                    url,
+                    occurrence,
+                    prefix === index.contentPackage ?
+                        `hardcoded absolute URL into this package's own ` +
+                            `address — write the package-relative ` +
+                            `"${rest}/", which the landing resolves ` +
+                            `against the site so the page follows the mount`
+                    :   `hardcoded absolute URL into package "${prefix}" ` +
+                            `— resolve it through that package's link ` +
+                            `manifest, whose entries carry the address, so a ` +
+                            `relocation does not leave this page behind`,
+                );
             } else if (shape === "rooted" && kind === "url") {
                 const rest =
                     prefix ? segments.slice(1).join("/") : segments.join("/");
@@ -497,9 +590,21 @@ export function auditHomepageLinks(index) {
                     url,
                     url,
                     occurrence,
-                    `url "${url}" is root-relative, but a landing's url: is ` +
-                        `resolved against the site — write "${rest}/", or ` +
-                        `href: for an address that is already resolved`,
+                    // A `url:` is package-relative by construction, so it
+                    // cannot address anything outside this package at all —
+                    // there is no relative spelling of another package's root.
+                    // `href:` is the field for an address already resolved.
+                    !rest ?
+                        `url "${url}" addresses ` +
+                            (prefix ?
+                                `package "${prefix}"'s landing`
+                            :   `the site root`) +
+                            `, but a landing's url: is package-relative and ` +
+                            `cannot leave this package — write ` +
+                            `href: "${url}", which is used verbatim`
+                    :   `url "${url}" is root-relative, but a landing's url: ` +
+                            `is resolved against the site — write "${rest}/", ` +
+                            `or href: for an address that is already resolved`,
                 );
             }
 

package/engine/content-lint.mjs

@@ -22,13 +22,18 @@
  * disagree without anything detecting it, which the canonical-separator
  * handling already did once on each side.
  *
- * Two rules, both about a note's identity:
+ * Three rules, all about a note's identity:
  *
  * 1. **Shape** — a `shortcode` is strictly ASCII-alphanumeric. It is the
  *    identity key referenced from saved world data, and it is half of the
  *    `type-shortcode` address, whose parse depends on the separating hyphen
  *    being the only hyphen in the string.
  * 2. **Uniqueness** — `(type, shortcode)` names one note.
+ * 3. **The package's own address** — exactly one note claims `/<package>/`,
+ *    which is {@link checkHomepageCount} (#52). It belongs here for the same
+ *    reason the other two do: it is a statement about which note holds which
+ *    address, it needs no `site:` configuration to decide, and a package with
+ *    no front page is misconfigured whether or not anyone runs a site build.
  *
  * **Nothing here writes.** A check reports and an author fixes.
  *
@@ -56,6 +61,7 @@ import path from "node:path";
 
 import { positionInFrontmatter } from "./diagnostics.mjs";
 import { walkMarkdownTree } from "./helpers.mjs";
+import { checkHomepageCount, isHomepage } from "./homepage.mjs";
 
 /**
  * The shape every `shortcode` must match: ASCII letters and digits only.
@@ -122,11 +128,16 @@ function collectNotes(contentBase, { skipDirectories } = {}) {
  * @param {object} [opts]
  * @param {readonly string[]} [opts.skipDirectories] - Directory names the walk
  *   ignores. Defaults to the configured list.
+ * @param {string} [opts.contentPackage] - The package this tree builds, for the
+ *   homepage rule. Dropped from that finding when unknown rather than guessed.
  * @returns {{findings: Array<{file: string, line?: number, column?: number,
  *   severity: "error"|"warning", message: string}>, notes: number,
  *   keys: number}} The findings, and what was inspected to produce them.
  */
-export function lintContentTree(contentBase, { skipDirectories } = {}) {
+export function lintContentTree(
+    contentBase,
+    { skipDirectories, contentPackage } = {},
+) {
     const findings = [];
     const notes = collectNotes(contentBase, { skipDirectories });
 
@@ -184,6 +195,16 @@ export function lintContentTree(contentBase, { skipDirectories } = {}) {
         return { findings, notes: 0, keys: 0 };
     }
 
+    // Deliberately after that return: a tree nobody has established exists has
+    // no homepage either, and saying so is noise about the second problem when
+    // the first is "check that the content tree is present".
+    findings.push(
+        ...checkHomepageCount(
+            notes.filter((n) => isHomepage(n.fm)),
+            { contentBase, contentPackage },
+        ),
+    );
+
     for (const [key, files] of byKey) {
         if (files.length < 2) continue;
         // Reported once per offending note rather than once per key: each note