@heroiclands/package-build 17.1.0 → 18.0.0

package/engine/content-links.mjs

@@ -23,7 +23,11 @@
  *    link to an anchor nobody declares compiles cleanly, emits an enricher, and
  *    dead-ends for the reader.
  * 2. **A dead address.** Every link is an address, and one resolving to no note
- *    is a typo. So is a target that does not parse as an address at all.
+ *    is a typo. So is a target that does not parse as an address at all. A
+ *    written target is a *partial* address — the segments it omits are
+ *    wildcards, and the package it omits is this one — so a target resolving to
+ *    *several* notes is an ambiguity rather than a first match, and is reported
+ *    naming every candidate (#59).
  * 3. **An unlabelled link.** `[[x]]` addresses nothing: the alias namespace it
  *    used to name is retired (#180), and a shortcode is an address rather than
  *    prose, so the link has neither a resolvable target nor text to show. The
@@ -55,17 +59,19 @@ import path from "node:path";
 
 import { matchAllOutsideCode } from "./code-fences.mjs";
 import { expandContentTables } from "./content-tables.mjs";
-import { walkMarkdownTree } from "./helpers.mjs";
+import { collectAnchors } from "./anchors.mjs";
+// The corpus, and everything derived from it, read from the one place that
+// derives it (#243). Nothing in the index's own import graph reaches this
+// module, so this is a plain static import rather than the deferred one
+// `sql-tables` needs to keep out of the compilers' cycle.
+import { authoredFrontmatter, indexRecordsFor, isNoteRecord, noteFile } from "./content-index.mjs";
 import { hasDocEntry } from "./item-docs.mjs";
-import { contentPackage } from "./content-package.mjs";
+import { NO_SYSTEM, systemOf } from "./document-subtypes.mjs";
+import { KNOWN_DOCUMENT_SUBTYPE_MAPS } from "./note-claims.mjs";
+import { loadPackConfig } from "./pack-config.mjs";
 import { searchableFrontmatter } from "./note-package.mjs";
-import {
-    canonicalKey,
-    loadForeignManifests,
-    manifestsComplete,
-    PACKAGE_BASE,
-    readCanonicalKey,
-} from "./kb-manifest.mjs";
+import { canonicalKey, PACKAGE_BASE, readCanonicalKey } from "./content-address.mjs";
+import { loadForeignIndexes } from "./metadata-index.mjs";
 import { frontmatterWikilinks, slugify } from "./web-wikilinks.mjs";
 import { homepageAddresses, isHomepage } from "./homepage.mjs";
 import { RETIRED_TYPES } from "./ids.mjs";
@@ -75,20 +81,46 @@ import { readQualifier } from "./wikilinks.mjs";
 /**
  * Every `{#anchor}` a note declares on a heading.
  *
+ * **Read from the content index's reader, not a second one.** This module kept
+ * its own until #243, and the two disagreed: it matched `{#([a-z0-9-]+)}` while
+ * {@link module:engine/content-index.collectAnchors} matches `{#([^}]+)}`, so
+ * an anchor with a capital in it — `{#CalendarFormat}` — existed for the index
+ * and for the compiler and did not exist for the link checker. Nothing links to
+ * one today, so the disagreement was latent; the first link to one would have
+ * been reported dead against a heading plainly present in the file.
+ *
+ * The specification puts no charset on the id: "`#id` represents an id anchor
+ * named `id`". The narrower pattern was this module's invention, which is the
+ * argument for there being one reader rather than a well-chosen one.
+ *
  * @param {string} body - The note's markdown body.
  * @returns {Set<string>} The declared anchor slugs.
  */
 export function anchorsOf(body) {
-    const found = new Set();
-    for (const line of String(body ?? "").split("\n")) {
-        const m = /^#{1,6}\s+.*\{#([a-z0-9-]+)\}\s*$/.exec(line.trim());
-        if (m) found.add(m[1]);
-    }
-    return found;
+    return new Set(collectAnchors(body).map((anchor) => anchor.slug));
 }
 
 /**
- * Load a content tree and build the index a link resolves against.
+ * Read a content tree into the index a link resolves against.
+ *
+ * **The corpus comes from the content index, not from a walk of this module's
+ * own** (#243). Every pass used to answer "which files are the content?" for
+ * itself and throw the answer away; this one now reads
+ * {@link module:engine/content-index.indexRecordsFor}, which is the same
+ * derivation the published artifact and the compilers are driven from. So a
+ * note the index records is a note the link check sees, and the addresses and
+ * anchors it resolves against are the ones every other pass will emit — rather
+ * than a second derivation that agrees with them only by inspection. That was
+ * not hypothetical: this module carried its own anchor reader until the anchor
+ * half of #243, and the two disagreed about which anchors existed.
+ *
+ * **The file is opened for its bytes and nothing else.** The index deliberately
+ * carries no note *body*, and a link lives in the body — so each note is read
+ * once, here, for the prose. Everything *about* the note — its frontmatter, its
+ * addresses, its anchors — is already in the record, and none of it is derived
+ * a second time. That is one read per note rather than the two this module did
+ * before, since the walk read the file and it then read it again for the raw
+ * text.
  *
  * The index mirrors what both builds construct, including the two addresses a
  * doc-carrying note answers to: `type/shortcode` for the document, and
@@ -99,57 +131,93 @@ export function anchorsOf(body) {
  *
  * @param {string} contentBase - Root of the content tree.
  * @param {object} [opts]
- * @param {string} [opts.manifestDir] - Where vendored foreign manifests live.
- *   Omitted, no cross-package address resolves.
- * @param {readonly string[]} [opts.skipDirectories] - Passed to the walk.
+ * @param {object} [opts.config] - The resolved build configuration, whose
+ *   fetched dependency indexes foreign addresses resolve through (#239), and
+ *   whose `contentPackage` every local address is built from. Omitted, the
+ *   ambient configuration is resolved and no cross-package address resolves.
+ * @param {readonly string[]} [opts.skipDirectories] - The walk's scope, passed
+ *   on to the index rather than defaulted away.
+ * @param {Map<string, object[]>} [opts.sqlTables] - Prepared `sql` results, by
+ *   note path.
+ * @param {object[]} [opts.records] - Index records the caller already derived,
+ *   so a command that also needs them — every one of them does, to answer its
+ *   `sql` tables — enumerates the corpus once rather than twice.
+ * @param {object[]} [opts.problems] - Collects the notes the index cannot
+ *   record, as diagnostics, instead of letting one of them abort the check
+ *   before it has reported anything else.
  * @returns {object} The notes, the index, and the resolvers built over it.
  */
-export function buildLinkIndex(contentBase, { manifestDir, skipDirectories } = {}) {
+export function buildLinkIndex(
+    contentBase,
+    { config, skipDirectories, sqlTables, records, problems } = {},
+) {
     const notes = [];
     const frontmatterLinks = [];
-    const walkOpts = skipDirectories ? { skipDirectories } : undefined;
 
-    for (const { frontmatter: fm, absPath } of walkMarkdownTree(contentBase, walkOpts)) {
-        if (!fm || typeof fm.type !== "string") continue;
+    // The one package every note in this tree belongs to. Taken from the
+    // configuration this build resolved — never from a note (`package:` is
+    // retired, so there is no second source an address could disagree with,
+    // #56) and never from the ambient one, which is a different configuration
+    // whenever a test injects one, `PACKAGE_BUILD_CONFIG` names one, or the
+    // command runs from a worktree (#243).
+    const resolved = config ?? loadPackConfig();
+    const pkg = resolved.contentPackage;
+
+    const indexRecords =
+        records ?? indexRecordsFor({ contentBase, config: resolved, skipDirectories, problems });
+
+    const byKey = new Map();
+    const anchors = new Map();
+
+    for (const record of indexRecords) {
+        // A documentation journal has a record of its own but no file and no
+        // authored frontmatter — it is a document this tree emits, not a note
+        // in it. Its addresses are keyed below, from the note it documents.
+        if (!isNoteRecord(record) || typeof record.type !== "string") continue;
+
+        const fm = authoredFrontmatter(record);
+        const rel = record.file.path;
+        const absPath = noteFile(contentBase, record);
         // The raw text is kept beside the parsed body: a consumer's own checks
         // may need what frontmatter carried, which the body has dropped.
         const raw = fs.readFileSync(absPath, "utf8");
         const body = raw.replace(/^---\n[\s\S]*?\n---\n?/, "");
-        const note = {
-            file: absPath,
-            rel: path.relative(contentBase, absPath).split(path.sep).join("/"),
-            fm,
-            body,
-            raw,
-            type: fm.type.toLowerCase(),
-        };
+        const type = record.type.toLowerCase();
+        const note = { file: absPath, rel, fm, body, raw, type };
+
         for (const hit of frontmatterWikilinks(fm)) {
             frontmatterLinks.push({ note, ...hit });
         }
         notes.push(note);
-    }
 
-    const byKey = new Map();
+        // The anchors the index recorded, rather than a second reading of the
+        // same headings — the disagreement #243's anchor half removed.
+        anchors.set(note, new Set((record.anchors ?? []).map((a) => a.slug)));
 
-    // The one package every note in this tree belongs to. Taken from the
-    // configuration, never from a note: `package:` is retired, so there is no
-    // second source an address could disagree with (#56).
-    const pkg = contentPackage();
-
-    for (const note of notes) {
-        const { fm, type } = note;
         if (typeof fm.shortcode === "string" && fm.shortcode) {
             byKey.set(`${type}/${fm.shortcode}`.toLowerCase(), note);
             // The canonical, fully qualified address alongside the short one,
             // so a package-qualified link checks the same way a bare one does.
-            byKey.set(canonicalKey(pkg, type, fm.shortcode), note);
+            // Taken from the record, which is where the address rule is applied
+            // once for the whole build.
+            const canonical =
+                record.address?.canonical ??
+                canonicalKey(pkg, systemOf(type, KNOWN_DOCUMENT_SUBTYPE_MAPS), type, fm.shortcode);
+            byKey.set(canonical, note);
             if (hasDocEntry(type)) {
                 byKey.set(`doc${type}/${fm.shortcode}`.toLowerCase(), note);
-                byKey.set(canonicalKey(pkg, `doc${type}`, fm.shortcode), note);
+                // A documentation journal is `none`: no game system defines a
+                // JournalEntry, and one note has one of them however many
+                // system blocks it carries.
+                byKey.set(canonicalKey(pkg, NO_SYSTEM, `doc${type}`, fm.shortcode), note);
             }
         }
     }
 
+    // The types a link may name, which are the ones notes declare. A
+    // documentation journal's `doc<type>` is deliberately not among them: it is
+    // virtual, and `readQualifier` resolves it from the base type rather than
+    // from a type any tree declares.
     const types = new Set(notes.map((n) => n.type));
 
     // A foreign package may use a type this tree has never seen, so its types
@@ -157,8 +225,8 @@ export function buildLinkIndex(contentBase, { manifestDir, skipDirectories } = {
     // is never checked at all.
     const localPackages = new Set([pkg]);
     const foreign =
-        manifestDir ?
-            loadForeignManifests(manifestDir, localPackages)
+        config ?
+            loadForeignIndexes(config, localPackages)
         :   { index: new Map(), packages: new Set(), stale: [] };
     for (const v of foreign.index.values()) if (v.type) types.add(v.type);
 
@@ -174,8 +242,6 @@ export function buildLinkIndex(contentBase, { manifestDir, skipDirectories } = {
         folder: path.dirname(n.rel).split("/").pop(),
     }));
 
-    const anchors = new Map(notes.map((n) => [n, anchorsOf(n.body)]));
-
     /**
      * Every wikilink in a note body, with its `dataview` tables expanded.
      *
@@ -187,13 +253,17 @@ export function buildLinkIndex(contentBase, { manifestDir, skipDirectories } = {
      */
     function linksOf(note) {
         let body = note.body;
-        if (/^[ \t]*(?:`{3,}|~{3,})[ \t]*dataview\b/im.test(body)) {
+        if (/^[ \t]*(?:`{3,}|~{3,})[ \t]*(?:dataview|sql)\b/im.test(body)) {
             body = expandContentTables(body, {
                 // Unfiltered: every note in the tree is this package's, so
                 // there is no other package's note to exclude (#56).
                 docs: tableDocs,
                 linkable: (d) => Boolean(d.fm.shortcode),
                 source: note.file,
+                // A `sql` table's links are checked like an authored one's, so
+                // its rows are prepared ahead of this walk — see
+                // {@link module:engine/sql-tables.prepareTreeSqlTables}.
+                sqlTables: sqlTables?.get(note.file),
             }).markdown;
         }
         const out = [];
@@ -232,25 +302,75 @@ export function buildLinkIndex(contentBase, { manifestDir, skipDirectories } = {
      * @param {string} target - The link target, anchor already removed.
      * @returns {object|undefined} The note it addresses.
      */
+    /**
+     * Every indexed entry an address names, matching only the segments it
+     * supplies.
+     *
+     * This is the whole of #59's resolution rule in one place: a written
+     * address is a *partial* one, unsupplied segments are wildcards, and the
+     * caller requires exactly one hit. Nothing here decides an ambiguity — zero
+     * and many are different findings with different fixes, so the count is
+     * returned rather than collapsed.
+     *
+     * The **system** is wildcarded unless stated. Defaulting it to `none` would
+     * exclude every link to an item, which is most of them.
+     *
+     * @param {Array<[string, any]>} pairs - Indexed `[canonicalKey, value]`.
+     * @param {object} q - The parsed qualifier.
+     * @returns {Array<[string, any]>} The matching pairs.
+     */
+    function matchAddress(pairs, q) {
+        const type = String(q.type).toLowerCase();
+        const shortcode = String(q.shortcode).toLowerCase();
+        return pairs.filter(([k]) => {
+            const parts = readCanonicalKey(k);
+            if (!parts) return false;
+            if (q.package && parts.package !== String(q.package).toLowerCase()) return false;
+            if (q.system && parts.system !== String(q.system).toLowerCase()) return false;
+            return parts.type === type && parts.shortcode === shortcode;
+        });
+    }
+
+    /**
+     * The local notes an address names, by the same rule.
+     *
+     * @param {object} q - The parsed qualifier.
+     * @returns {object[]} The notes.
+     */
+    function matchLocal(q) {
+        return matchAddress([...byKey], q).map(([, v]) => v);
+    }
+
     function resolveAddress(target) {
         const qualified = readQualifier(target, types, packages);
         if (!qualified || qualified.reason) return undefined;
-        return byKey.get(
-            qualified.package ?
-                canonicalKey(qualified.package, qualified.type, qualified.shortcode)
-            :   `${qualified.type}/${qualified.shortcode}`.toLowerCase(),
-        );
+        // A target naming neither package nor system stays the system-blind
+        // short key, which is already the wildcard an author writing
+        // `[[skill-melee]]` means.
+        //
+        // Anything that *does* state one is matched by the segments it supplied
+        // rather than by an exact key (#59), and exactly one hit is required —
+        // two systems' documents legitimately share a
+        // `(package, type, shortcode)`, so a target that names no system may
+        // name two notes, and naming two is not resolving.
+        if (!qualified.package && !qualified.system) {
+            return byKey.get(`${qualified.type}/${qualified.shortcode}`.toLowerCase());
+        }
+        const hits = matchLocal(qualified);
+        return hits.length === 1 ? hits[0] : undefined;
     }
 
     /**
      * Every foreign manifest entry an address names, in package order.
      *
-     * A **package-qualified** address names at most one, by construction. An
-     * unqualified one names no package, so it resolves against any foreign one
-     * that publishes it — and only when exactly one does. Two claimants make it
-     * ambiguous, which is a different finding from resolving nowhere and has a
-     * different fix, so the count is returned rather than collapsed here
-     * (#184).
+     * A written target is a **partial** address, so this matches on the
+     * segments it supplies and wildcards the rest (#59). A target naming a
+     * package necessarily names its system too — omission runs left to right —
+     * so the fully qualified form matches at most one entry; a shorter one
+     * names no package, and resolves against any foreign package that
+     * publishes it. Either way only exactly one hit resolves. Two claimants make it ambiguous, which is a different finding
+     * from resolving nowhere and has a different fix, so the count is returned
+     * rather than collapsed here (#184).
      *
      * @param {string} target - The link target.
      * @returns {object[]} The foreign entries, each carrying its `package`.
@@ -258,18 +378,7 @@ export function buildLinkIndex(contentBase, { manifestDir, skipDirectories } = {
     function foreignHits(target) {
         const q = readQualifier(target, types, packages);
         if (!q || q.reason) return [];
-        if (q.package) {
-            const one = foreign.index.get(canonicalKey(q.package, q.type, q.shortcode));
-            return one ? [one] : [];
-        }
-        const type = String(q.type).toLowerCase();
-        const shortcode = String(q.shortcode).toLowerCase();
-        return [...foreign.index]
-            .filter(([k]) => {
-                const parts = readCanonicalKey(k);
-                return parts?.type === type && parts.shortcode === shortcode;
-            })
-            .map(([, v]) => v);
+        return matchAddress([...foreign.index], q).map(([, v]) => v);
     }
 
     /**
@@ -299,7 +408,6 @@ export function buildLinkIndex(contentBase, { manifestDir, skipDirectories } = {
          */
         contentPackage: pkg,
         foreign,
-        manifests: manifestsComplete(localPackages, foreign.packages),
         linksOf,
         /**
          * Resolve a link target the way both builds do, or `undefined`. Every
@@ -338,17 +446,17 @@ const SITE_HOST = /^(?:[a-z0-9-]+\.)*heroiclands\.org$/i;
  * is that a landing therefore cannot be addressed. It does not follow: a
  * landing's address is not a *note's* address but the **package's**, and
  * {@link PACKAGE_BASE} already records where each package is served. That is a
- * frozen constant vendored into every repository, so consulting it walks no
+ * frozen constant compiled into every build, so consulting it walks no
  * tree, reads no manifest and builds no index — which is precisely why the
  * mechanism survives `homepage` mode, where the licensing fence means none of
  * those exist.
  *
  * The roster is consulted **for landings only**. Widening the package set the
  * other rules read would make them offer manifest-based advice about packages
- * no manifest is vendored for.
+ * no index has been fetched for.
  *
  * @param {string} ownPackage - The package this build publishes.
- * @param {Iterable<string>} manifestPackages - Packages a vendored manifest
+ * @param {Iterable<string>} manifestPackages - Packages a fetched index
  *   names, which are addressable whether or not the roster lists them.
  * @returns {Map<string, string>} Package to base, each base slash-terminated.
  */
@@ -458,7 +566,7 @@ function readAddress(url, packages) {
  *   and what replaced it, so this is a fact rather than a guess — and it is
  *   exactly the SoHL defect.
  * - A **hardcoded absolute URL** into this package's own prefix, or into one a
- *   vendored manifest names. Every one of them has a better form to write, which
+ *   a fetched index names. Every one of them has a better form to write, which
  *   is why every one is reported — including a bare `/<package>/`, which names
  *   another package's landing (#87).
  *
@@ -533,7 +641,7 @@ export function auditHomepageLinks(index) {
 
             // Landings first, and by the roster rather than by the manifest
             // package set: a landing is addressable in a repository that
-            // vendors no manifest at all, which is the case the fence creates
+            // has fetched no index at all, which is the case the fence creates
             // and the case this rule exists for (#87).
             const landing = landingTarget(url, bases);
             if (landing) {