@heroiclands/package-build 17.1.0 → 18.0.0

package/engine/content-lint.mjs

@@ -22,7 +22,7 @@
  * disagree without anything detecting it, which the canonical-separator
  * handling already did once on each side.
  *
- * Three rules, all about a note's identity:
+ * Four rules, all about a note's identity:
  *
  * 1. **Shape** — a `shortcode` is strictly ASCII-alphanumeric. It is the
  *    identity key referenced from saved world data, and it is half of the
@@ -34,6 +34,11 @@
  *    reason the other two do: it is a statement about which note holds which
  *    address, it needs no `site:` configuration to decide, and a package with
  *    no front page is misconfigured whether or not anyone runs a site build.
+ * 4. **Vacated addresses** — a `renamedFrom:` entry names an address this note
+ *    used to hold and nothing holds now (#278). It is the same statement as
+ *    rule 2 read backwards, and it needs the same whole-tree view: an entry can
+ *    only be checked against every *other* note's address, and two notes
+ *    claiming one predecessor is the uniqueness rule applied to the past.
  *
  * **Nothing here writes.** A check reports and an author fixes.
  *
@@ -61,8 +66,11 @@ import path from "node:path";
 
 import { ADDRESS_SEGMENT_PATTERN } from "./address-charset.mjs";
 import { positionInFrontmatter } from "./diagnostics.mjs";
-import { walkMarkdownTree } from "./helpers.mjs";
+import { assertStatedScope } from "./helpers.mjs";
+// The corpus, read from the one pass that derives it (#243).
+import { authoredFrontmatter, indexRecordsFor, isNoteRecord, noteFile } from "./content-index.mjs";
 import { checkHomepageCount, isHomepage } from "./homepage.mjs";
+import { declaresRenamedFrom, renamedFrom, renamedFromEntries } from "./note-renames.mjs";
 
 /**
  * The shape every `shortcode` must match: ASCII letters and digits only.
@@ -98,33 +106,164 @@ export function isValidShortcode(value) {
 }
 
 /**
- * Collect the notes a lint pass reasons about.
+ * Collect the notes a lint pass reasons about, from the content index.
  *
  * Only notes carrying a `type` are content notes. Vault scaffolding —
  * `Templates/`, a `README`, a repository's own `CLAUDE.md` — has no type, is
  * neither addressed nor addressable, and would fail rules it can never satisfy.
  *
+ * **Read from the index, not from a walk of this pass's own** (#243). The
+ * `lint` command already derives the index — its link check and its `sql`
+ * tables are built from it — and then walked the tree a second time to get
+ * here, so one command held two answers to "which files are the corpus?" and
+ * compared findings drawn from both. It now holds one: the records are derived
+ * once by the command and handed to every pass, this one included.
+ *
+ * The frontmatter is the note's own, recovered with
+ * {@link module:engine/content-index.authoredFrontmatter} — a lint of what an
+ * author wrote must not be handed the keys the index derived, or it would
+ * report `address:` and `anchors:` as fields nobody may write.
+ *
  * @param {string} contentBase - Root of the content tree.
  * @param {object} [opts]
- * @param {readonly string[]} [opts.skipDirectories] - Passed to the walk.
+ * @param {readonly string[]} [opts.skipDirectories] - The corpus scope, stated
+ *   by the caller.
+ * @param {object} [opts.config] - The resolved build configuration.
+ * @param {readonly object[]} [opts.records] - Records the caller derived.
+ * @param {object[]} [opts.problems] - Collects the notes the index cannot
+ *   record, so one of them does not silence the lint.
  * @returns {Array<{fm: object, absPath: string, file: string}>} The notes, in
  *   path order so findings read top to bottom.
  */
-function collectNotes(contentBase, { skipDirectories } = {}) {
+function collectNotes(contentBase, { skipDirectories, config, records, problems } = {}) {
     const notes = [];
-    const walkOpts = skipDirectories ? { skipDirectories } : undefined;
-    for (const { frontmatter: fm, absPath } of walkMarkdownTree(contentBase, walkOpts)) {
-        if (!fm || !fm.type) continue;
+    const corpus =
+        records ??
+        (assertStatedScope(skipDirectories, "collectNotes"),
+        fs.existsSync(contentBase) ?
+            indexRecordsFor({ contentBase, config, skipDirectories, problems })
+        :   []);
+
+    for (const record of corpus) {
+        // A documentation journal is a document this tree emits, not a note in
+        // it: it has no authored frontmatter for a lint to reason about.
+        if (!isNoteRecord(record) || !record.type) continue;
+        const absPath = noteFile(contentBase, record);
         notes.push({
-            fm,
+            fm: authoredFrontmatter(record),
             absPath,
             file: path.relative(process.cwd(), absPath),
         });
     }
+    // The records already come in content-path order, so this only re-states
+    // the guarantee findings depend on: they read top to bottom.
     notes.sort((a, b) => (a.absPath < b.absPath ? -1 : 1));
     return notes;
 }
 
+/**
+ * What one note's `renamedFrom:` says, checked against itself (#278).
+ *
+ * The entries a note can be wrong about on its own: a value that is not a
+ * shortcode, one naming the address the note holds *now*, one written twice.
+ * Every one of them is silent without a check — a malformed entry is skipped by
+ * the diff, so the author who wrote it sees the rename they were trying to
+ * announce reported as a withdrawal anyway, with nothing saying why.
+ *
+ * The cross-note questions are not here, because one note cannot answer them:
+ * whether an entry names an address some *other* note still publishes, and
+ * whether two notes claim one predecessor, both need the whole tree and are
+ * asked in {@link lintContentTree} once it has one.
+ *
+ * @param {object} note - The note, as {@link collectNotes} yields it.
+ * @param {() => string} raw - Reads the file, deferred so a clean note costs
+ *   nothing.
+ * @returns {Array<object>} The findings.
+ */
+function checkRenamedFrom({ fm, file }, raw) {
+    if (!declaresRenamedFrom(fm)) return [];
+    const findings = [];
+    const at = (value) => positionInFrontmatter(raw(), "renamedFrom", value);
+    const shortcode = typeof fm.shortcode === "string" ? fm.shortcode.trim() : "";
+
+    // A rename is a statement about where this note's address moved *to*, so a
+    // note with no address of its own has made no such statement. Reported
+    // before the entries: telling the author their entries are fine would be
+    // the less useful half of the answer.
+    if (!shortcode) {
+        return [
+            {
+                file,
+                ...at(undefined),
+                severity: "error",
+                message:
+                    "`renamedFrom` names the address this note used to hold, " +
+                    "but the note declares no `shortcode`, so it holds none now " +
+                    "and nothing was renamed",
+            },
+        ];
+    }
+
+    const seen = new Set();
+    for (const entry of renamedFromEntries(fm)) {
+        // Not `String(entry)`: the point is that the author wrote something
+        // that is not a shortcode, and rendering a list as `a,b` would show
+        // them a string they never typed.
+        if (typeof entry !== "string" || !entry.trim()) {
+            findings.push({
+                file,
+                ...at(undefined),
+                severity: "error",
+                message:
+                    `\`renamedFrom\` takes shortcodes, and one entry is ` +
+                    `${entry === "" || (typeof entry === "string" && !entry.trim()) ? "blank" : `a ${typeof entry}`}; ` +
+                    `it is skipped, so the rename it was meant to announce is ` +
+                    `still reported as a withdrawal`,
+            });
+            continue;
+        }
+        const value = entry.trim();
+        if (!isValidShortcode(value)) {
+            findings.push({
+                file,
+                ...at(value),
+                severity: "error",
+                message:
+                    `\`renamedFrom: ${value}\` is not strictly alphanumeric, so ` +
+                    `it is not an address this package ever published — a ` +
+                    `shortcode is held to one charset whether it is current or past`,
+            });
+            continue;
+        }
+        if (value === shortcode) {
+            findings.push({
+                file,
+                ...at(value),
+                severity: "error",
+                message:
+                    `\`renamedFrom: ${value}\` is this note's own shortcode, so ` +
+                    `it declares a rename from itself; name the shortcode it ` +
+                    `was published under before, or drop the key`,
+            });
+            continue;
+        }
+        if (seen.has(value)) {
+            findings.push({
+                file,
+                ...at(value),
+                // The declaration still works — the reader de-duplicates — so
+                // this is tidiness, and failing a build over it would red a
+                // tree whose renames are all correctly announced.
+                severity: "warning",
+                message: `\`renamedFrom: ${value}\` is listed twice; the repeat says nothing new`,
+            });
+            continue;
+        }
+        seen.add(value);
+    }
+    return findings;
+}
+
 /**
  * Lint every address in a content tree.
  *
@@ -134,26 +273,58 @@ function collectNotes(contentBase, { skipDirectories } = {}) {
  *   ignores. Defaults to the configured list.
  * @param {string} [opts.contentPackage] - The package this tree builds, for the
  *   homepage rule. Dropped from that finding when unknown rather than guessed.
+ * @param {object} [opts.config] - The resolved build configuration, which the
+ *   corpus is derived against.
+ * @param {readonly object[]} [opts.records] - Index records the caller already
+ *   derived, so a command reads one corpus (#243).
+ * @param {object[]} [opts.problems] - Collects the notes the index cannot
+ *   record, instead of letting one of them silence the lint.
  * @returns {{findings: Array<{file: string, line?: number, column?: number,
  *   severity: "error"|"warning", message: string}>, notes: number,
  *   keys: number}} The findings, and what was inspected to produce them.
  */
-export function lintContentTree(contentBase, { skipDirectories, contentPackage } = {}) {
+export function lintContentTree(
+    contentBase,
+    { skipDirectories, contentPackage, config, records, problems } = {},
+) {
     const findings = [];
-    const notes = collectNotes(contentBase, { skipDirectories });
+    const notes = collectNotes(contentBase, { skipDirectories, config, records, problems });
 
     /** @type {Map<string, Array<{file: string, absPath: string}>>} */
     const byKey = new Map();
+    /** @type {Map<string, Array<{file: string, absPath: string, shortcode: string}>>} */
+    const claimedPredecessors = new Map();
 
-    for (const { fm, absPath, file } of notes) {
+    for (const note of notes) {
+        const { fm, absPath, file } = note;
         const shortcode = fm.shortcode;
-        // Folder documents and keyless entries carry no address at all.
-        if (!shortcode) continue;
 
         // Read only when there is something to say about the note, so a clean
         // tree costs one pass rather than two.
         const raw = () => fs.readFileSync(absPath, "utf8");
 
+        // Before the keyless `continue` below, because a note declaring a
+        // rename while carrying no address of its own is exactly one of the
+        // things this reports — and reaching it after the skip would mean it
+        // never ran on the case that needs it most.
+        findings.push(...checkRenamedFrom(note, raw));
+        if (shortcode) {
+            // The de-duplicated reader, not the raw entries: a note that listed
+            // one predecessor twice has made one claim, and indexing it twice
+            // would make the note collide with itself and be reported as two
+            // notes claiming one address.
+            for (const value of renamedFrom(fm)) {
+                if (value === shortcode) continue;
+                const claim = `${fm.type}:${value}`;
+                const seen = claimedPredecessors.get(claim);
+                if (seen) seen.push({ file, absPath, shortcode });
+                else claimedPredecessors.set(claim, [{ file, absPath, shortcode }]);
+            }
+        }
+
+        // Folder documents and keyless entries carry no address at all.
+        if (!shortcode) continue;
+
         const key = `${fm.type}:${shortcode}`;
         const seen = byKey.get(key);
         if (seen) seen.push({ file, absPath });
@@ -230,5 +401,52 @@ export function lintContentTree(contentBase, { skipDirectories, contentPackage }
         }
     }
 
+    // The two questions about a declared rename that need the whole tree
+    // (#278). Both are the uniqueness rule above, applied to the past: an
+    // address has one holder, so it has one successor and it cannot be both
+    // vacated and occupied.
+    for (const [claim, claimants] of claimedPredecessors) {
+        const live = byKey.get(claim);
+        if (live) {
+            for (const { file, absPath } of claimants) {
+                findings.push({
+                    file,
+                    ...positionInFrontmatter(
+                        fs.readFileSync(absPath, "utf8"),
+                        "renamedFrom",
+                        claim.slice(claim.indexOf(":") + 1),
+                    ),
+                    severity: "error",
+                    message:
+                        `\`renamedFrom\` claims "${claim}", which ` +
+                        `${live.map((f) => f.file).join(", ")} still publishes; ` +
+                        `that address was never vacated, so nothing was renamed ` +
+                        `away from it`,
+                });
+            }
+        }
+        if (claimants.length < 2) continue;
+        // Named on every claimant rather than once on the address, for the same
+        // reason a duplicate address is: each is a file an author has to open,
+        // and a finding naming only the address sends them hunting for the rest.
+        for (const { file, absPath, shortcode } of claimants) {
+            const others = claimants.filter((c) => c.file !== file);
+            findings.push({
+                file,
+                ...positionInFrontmatter(
+                    fs.readFileSync(absPath, "utf8"),
+                    "renamedFrom",
+                    claim.slice(claim.indexOf(":") + 1),
+                ),
+                severity: "error",
+                message:
+                    `"${claim}" is claimed as a predecessor by more than one ` +
+                    `note — this one (now "${shortcode}") and ` +
+                    `${others.map((c) => `${c.file} (now "${c.shortcode}")`).join(", ")}; ` +
+                    `an address had one holder, so it has one successor`,
+            });
+        }
+    }
+
     return { findings, notes: notes.length, keys: byKey.size };
 }

package/engine/content-package.mjs

@@ -28,7 +28,8 @@ import { loadPackConfig } from "./pack-config.mjs";
  * The **content** package: the distribution unit this repository's notes belong
  * to, and the **address namespace** every one of them is published under.
  *
- * It is the first segment of every canonical key (`sohl-skill-clmb`), the name
+ * It is the first segment of every canonical key
+ * (`package-system-type-shortcode`, so `sohl-none-doc-gear`), the name
  * of the link manifest this build emits (`sohl.json`), and the package a
  * cross-package wikilink writes to reach one of these notes. So it is the
  * repository's identity in the address space, not a switch — and never dead

package/engine/content-tables.mjs

@@ -23,7 +23,7 @@
  * TABLE WITHOUT ID
  *   link(file.path, name.full) AS "Name",
  *   sohl.weight AS "Weight"
- * WHERE type = "armorgear" and sohl.material = "Cloth"
+ * WHERE type = "armor" and sohl.material = "Cloth"
  * SORT name.full ASC
  * ```
  *
@@ -1087,11 +1087,42 @@ export function renderContentTable(spec, rows, linkable, self) {
  *   body can name an authored position (#17). An `errors` entry carries the
  *   0-based line of the directive that failed, for the same reason.
  */
+/**
+ * The `WHERE` clause of a query, as authored, for a message that has to name
+ * what matched nothing.
+ *
+ * The parsed spec holds an expression tree, and rendering that back to text
+ * would be a second dialect of the query language to keep in step. The authored
+ * line is what the author wrote and what they will edit, so it is what the
+ * finding quotes.
+ *
+ * @param {string} query - The fence's contents.
+ * @returns {string} The clause, or "" when the query has none.
+ */
+function whereText(query) {
+    const match = /^\s*WHERE\s+(.+?)\s*$/im.exec(String(query ?? ""));
+    return match ? `\`${match[1]}\`` : "";
+}
+
 export function expandContentTables(
     markdown,
-    { docs = [], linkable = () => false, source = "", self = undefined } = {},
+    {
+        docs = [],
+        linkable = () => false,
+        source = "",
+        self = undefined,
+        sqlTables = undefined,
+    } = {},
 ) {
     const errors = [];
+    // How many `sql` directives have been seen, which is how a prepared
+    // result is found: the passes disagree about a body's leading whitespace,
+    // so a line number would not survive the trip, and an ordinal does.
+    let sqlOrdinal = 0;
+    // One entry per `dataview` directive still authored, so a caller can say how
+    // much of the corpus is still on the retiring language without this pass
+    // failing a build over it (#246).
+    const warnings = [];
     const lines = String(markdown ?? "").split("\n");
     const out = [];
     // Which authored line each emitted line came from, so a later pass can
@@ -1125,17 +1156,82 @@ export function expandContentTables(
         const closer = new RegExp(`^[ \\t]*${marker[0]}{${marker.length},}[ \\t]*$`);
         while (close < lines.length && !closer.test(lines[close])) close++;
         const isQuery = /^dataview\b/i.test(info.trim());
+        const isSql = /^sql\b/i.test(info.trim());
         const block = lines.slice(i, Math.min(close + 1, lines.length));
+
+        // A `sql` directive was run before this pass began — DuckDB is async and
+        // this is not (see `prepareSqlTables`). What is left here is splicing
+        // the rendered table in at the directive's own position.
+        if (isSql && close < lines.length) {
+            const prepared = sqlTables?.[sqlOrdinal];
+            sqlOrdinal += 1;
+            const failure =
+                !prepared ?
+                    "sql content table was not prepared — this pass was given no " +
+                    "`sqlTables`, so the query could not be run"
+                : prepared.reason ? prepared.reason
+                : prepared.rows === 0 && !prepared.allowEmpty ?
+                    "sql query selects no notes — write ```sql allow-empty if " + "that is intended"
+                :   undefined;
+            if (failure) {
+                errors.push({
+                    source,
+                    directive: block.join("\n"),
+                    reason: failure,
+                    line: i,
+                    column: indent.length + 1,
+                });
+                block.forEach((text, k) => emit(text, i + k));
+                i = close;
+                continue;
+            }
+            if (out.length > 0 && out[out.length - 1].trim() !== "") emit("", i, true);
+            for (const row of prepared.markdown.split("\n")) emit(`${indent}${row}`, i, true);
+            if (close + 1 < lines.length && lines[close + 1].trim() !== "") emit("", i, true);
+            i = close;
+            continue;
+        }
+
         if (!isQuery || close >= lines.length) {
             block.forEach((text, k) => emit(text, i + k));
             i = close;
             continue;
         }
         const query = lines.slice(i + 1, close).join("\n");
+        warnings.push({
+            source,
+            line: i,
+            column: indent.length + 1,
+            reason:
+                "`dataview` content tables are being replaced by `sql` over the " +
+                "content index (#246)",
+        });
+        // `dataview allow-empty` says a table selecting nothing is the intended
+        // state. Spelled on the fence rather than in the query, because it is a
+        // statement about this *directive* and not part of the query language.
+        const allowEmpty = /\ballow-empty\b/i.test(info);
         let table;
         try {
             const spec = parseDataviewQuery(query);
             const rows = selectRows(spec, docs, self);
+            // A table that selects nothing is almost always a stale query — a
+            // renamed type, a retired category, a typo'd path — and publishing
+            // a header with nothing under it makes that indistinguishable from
+            // a category that is legitimately empty. Eight tables in `sohl`'s
+            // `Rules/Bestiary.md` published that way for months after the
+            // `creature` → `being` rename, and no build said a word (#223).
+            if (rows.length === 0 && !allowEmpty) {
+                errors.push({
+                    source,
+                    directive: block.join("\n"),
+                    reason:
+                        `dataview query selects no notes` +
+                        `${whereText(query) ? `: nothing matches ${whereText(query)}` : ""}` +
+                        ` — write \`\`\`dataview allow-empty if that is intended`,
+                    line: i,
+                    column: indent.length + 1,
+                });
+            }
             table = renderContentTable(spec, rows, linkable, self);
         } catch (err) {
             errors.push({
@@ -1159,5 +1255,5 @@ export function expandContentTables(
         }
         i = close;
     }
-    return { markdown: out.join("\n"), errors, lineMap };
+    return { markdown: out.join("\n"), errors, warnings, lineMap };
 }

package/engine/document-subtypes.mjs

@@ -64,7 +64,10 @@
  * @module
  */
 
-import { assertTypeNotRetired } from "./ids.mjs";
+import { assertTypeNotRetired, currentType } from "./ids.mjs";
+import { NO_SYSTEM } from "./systems.mjs";
+
+export { NO_SYSTEM };
 import { locateFrontmatterKey } from "./retired-fields.mjs";
 
 /**
@@ -199,6 +202,13 @@ function frozenRow(system, noteType, row) {
 /**
  * The row a system declares for a note type, or nothing.
  *
+ * The **one** place a map is indexed, so the retirement window for a renamed
+ * type is honoured everywhere a row is asked for — {@link documentSubtype},
+ * {@link mapsNoteType}, {@link noteTypesFor}'s consumers and
+ * {@link systemOf} all arrive here. A note still spelling `armorgear` finds the
+ * `armor` row, compiles into the `armorgear` document it always did, and keeps
+ * the address it publishes at; only the *report* tells it to move (#78).
+ *
  * @param {DocumentSubtypeMap} map - The system's map.
  * @param {string|undefined} noteType - The note's declared `type`.
  * @returns {Readonly<DocumentSubtypeRow>|undefined} The row, or `undefined`
@@ -206,7 +216,7 @@ function frozenRow(system, noteType, row) {
  */
 export function subtypeRow(map, noteType) {
     if (!noteType || typeof noteType !== "string") return undefined;
-    return map?.types?.[noteType];
+    return map?.types?.[currentType(noteType)];
 }
 
 /**
@@ -339,9 +349,17 @@ export function documentSubtype(map, noteType, fm, { file, absPath } = {}) {
  *   whole point of the issue.
  *
  * A **retired** spelling is refused by name before any of that. Without it a
- * reference left behind by a rename would take the unmapped fallback and
- * address a document of the old name — resolving silently, which is precisely
- * what a retirement exists to stop (#78).
+ * reference left behind by a merge would take the unmapped fallback and address
+ * a document of the old name — resolving silently, which is precisely what a
+ * retirement exists to stop.
+ *
+ * A **renamed** spelling is the opposite case and resolves normally, through
+ * the same {@link currentType} every other lookup goes through (#78). It has to
+ * be this side as much as the note's own `type:`: the overwhelming majority of
+ * the 31,000 occurrences of the old names are `(type, shortcode)` references
+ * inside a being's `items:` list, so a window that resolved notes but not
+ * references would silently drop 30,000 embedded items rather than compile them
+ * unchanged.
  *
  * @param {DocumentSubtypeMap} map - The system's map.
  * @param {string|undefined} noteType - The type the reference names.
@@ -361,7 +379,9 @@ export function referencedSubtype(map, noteType, document) {
 
     const row = subtypeRow(map, noteType);
     // No row: the type is the consumer's own, and its document is stamped with
-    // the note type. See the note on the unmapped fallback above.
+    // the note type. See the note on the unmapped fallback above. Returned as
+    // *authored*, not normalised — the Item pass stamps an unmapped document
+    // with `fm.type` verbatim, and the two answers have to agree.
     if (!row) return { subType: noteType };
 
     if (row.document !== document) {
@@ -438,3 +458,26 @@ function list(values) {
     if (quoted.length <= 1) return quoted.join("");
     return `${quoted.slice(0, -1).join(", ")} or ${quoted[quoted.length - 1]}`;
 }
+
+/**
+ * Which system defines the document a note of this type compiles into.
+ *
+ * A journal, a macro, a scene and an item's documentation journal belong to no
+ * system and answer {@link NO_SYSTEM}; an item or an actor belongs to whichever
+ * system's map claims its note type.
+ *
+ * `maps` is required rather than defaulted, so this stays reachable from any
+ * pass: the registry of maps lives in `note-claims.mjs`, which imports half the
+ * engine, and defaulting to it here would drag that in wherever an address is
+ * built.
+ *
+ * @param {string} type - The note's `type`.
+ * @param {readonly object[]} maps - The document-subtype maps this build ships.
+ * @returns {string} The system id, or {@link NO_SYSTEM}.
+ */
+export function systemOf(type, maps) {
+    for (const map of maps ?? []) {
+        if (subtypeRow(map, type)) return map.system;
+    }
+    return NO_SYSTEM;
+}

package/engine/field-reference.mjs

@@ -170,7 +170,7 @@ function workedExample(type, fields) {
         // this example is the smallest note that compiles.
         "id: <16-character id>",
         "sohl:",
-        "  archetype: null",
+        "  templatePriority: null",
     ];
     for (const field of required) {
         lines.push(`  ${field.name}: <${field.shape ?? "value"}>`);