@heroiclands/package-build 17.2.0 → 18.1.0

package/engine/content-index.mjs

@@ -61,6 +61,28 @@
  * Foundry data root, so anything left there ships inside the installed system
  * to every player, and a build artifact has no business there.
  *
+ * **What it deliberately does not carry: the note's text, and positions within
+ * it.** #243 asks whether the index should record a position for every
+ * frontmatter key, so that a pass reading the index could report a field defect
+ * without opening the file. It should not, and the numbers are not close: over
+ * `sohl`'s 1,685 notes the index is 3.0 MB and holds 50,598 leaf values, so a
+ * `{line, column}` on each would add roughly 1.6 MB — **a 54% larger artifact**
+ * — to carry data that is only ever read on the *failing* path.
+ *
+ * The rule that replaces it is the one this module was already built on:
+ * **the index carries what is _about_ a note; the file carries the note's text
+ * and every position within it.** Any pass needing either opens the file whose
+ * path the record already names ({@link noteFile}). That costs nothing it was
+ * not already paying — a check reads each note once for its body, and a
+ * compiler must read the prose regardless, so while it holds the bytes a
+ * position is free. Recording positions would charge every build, and every
+ * reader of the artifact, for something the passes that want them get for
+ * nothing.
+ *
+ * The exception proves the rule: an **anchor** carries its `line`, because an
+ * anchor is a fact about the note's structure that a consumer addresses
+ * directly, not a locator for a diagnostic about a key.
+ *
  * @module
  */
 
@@ -69,111 +91,58 @@ import path from "node:path";
 
 import unidecode from "unidecode";
 
-import { addressSlug } from "./content-address.mjs";
-import { canonicalKey } from "./kb-manifest.mjs";
-import { entriesForNote, foundryIdentities } from "./manifest-emit.mjs";
-import { walkMarkdownTree } from "./helpers.mjs";
-import { loadPackConfig } from "./pack-config.mjs";
+import { metadataFileName } from "./metadata-index.mjs";
+import { addressSlug, canonicalKey } from "./content-address.mjs";
+// One reader for a note's anchors, shared with the link checker and with the
+// builds that emit a link (#243). Re-exported because this is where callers
+// have always addressed it.
+import { collectAnchors } from "./anchors.mjs";
+import { subtypeRow, NO_SYSTEM, systemOf } from "./document-subtypes.mjs";
+import { KNOWN_DOCUMENT_SUBTYPE_MAPS } from "./note-claims.mjs";
 
 /**
- * The keys this module adds to a record, which a note therefore may not carry
- * itself.
+ * The `<system>` a note belongs to when it belongs to none.
  *
- * `package` is the note's distribution unit — the configured `contentPackage`,
- * since a note declaring its own is a hard error (package-build#56) — and it
- * matches what the content-table expander puts on the same field, so a query
- * reads the same value from either. `file` namespaces the note's place in the
- * tree, again matching the expander's `file.*`.
+ * The specification's word, not this module's: the canonical address carries it
+ * in the same position — `harnadventures-none-being-grod` — so the index and
+ * the address say "no system" the same way (#59).
  *
- * Both are checked rather than assumed: `folder` is real frontmatter on most
- * notes, so the neighbouring names are close enough to a real key that a silent
- * overwrite is a plausible future rather than a hypothetical one.
- *
- * @type {ReadonlyArray<string>}
- */
-export const DERIVED_KEYS = Object.freeze([
-    "package",
-    "file",
-    "address",
-    "anchors",
-    "nameAscii",
-    "aliasesAscii",
-    "foundry",
-    "documentation",
-    "documents",
-]);
-
-/**
- * A heading, and the `{#slug}` anchor it declares.
- *
- * Kept identical to the pair {@link splitPages} matches, because the two must
- * agree about what an anchor is: that pass decides which sections become
- * addressable journal pages, and an index naming an anchor it does not produce
- * would advertise a link that resolves nowhere. `tests/content-index.test.ts`
- * asserts the two find the same anchors, so drift fails the suite rather than
- * shipping.
- */
-const HEADING = /^\s*(#{1,6})\s+(.+?)\s*#*\s*$/;
-const ANCHOR = /^(.*?)\s*\{#([^}]+)\}\s*$/;
-
-/**
- * The `{#slug}` anchors a note's body declares, with where each one sits.
- *
- * Only headings carrying an explicit anchor are collected. A bare `#` heading
- * also starts a journal page, but it declares no slug, so nothing can address
- * it with `#…` — listing it would offer a link that cannot be written.
- *
- * @param {string} body - The note's markdown body, frontmatter already removed.
- * @param {number} [bodyLine] - The 1-based file line the body starts on, from
- *   `parseMarkdownFile`. Anchors are reported at their position in the **file**,
- *   so an editor can jump straight to one; passing nothing numbers from the body.
- * @returns {Array<{slug: string, name: string, level: number, line: number}>}
- *   In document order.
+ * @type {string}
  */
-export function collectAnchors(body, bodyLine = 1) {
-    const anchors = [];
-    let inCodeBlock = false;
-    const lines = String(body ?? "").split("\n");
 
-    for (let i = 0; i < lines.length; i++) {
-        // A fenced block's contents are not headings, and `#` is a comment in
-        // most of what gets fenced.
-        if (lines[i].trim().startsWith("```")) {
-            inCodeBlock = !inCodeBlock;
-            continue;
-        }
-        if (inCodeBlock) continue;
-
-        const heading = HEADING.exec(lines[i]);
-        if (!heading) continue;
-        const anchor = ANCHOR.exec(heading[2].trim());
-        if (!anchor) continue;
+export { collectAnchors };
+import { entriesForNote, foundryIdentities } from "./foundry-entries.mjs";
+import { walkMarkdownTree } from "./helpers.mjs";
+import { resolveNoteId } from "./note-ids.mjs";
+// The retired-field refusal and the key locator, so a note authoring a derived
+// key is reported where it is rather than as a bare abort (#243).
+import { assertNoDeclaredPackage } from "./note-package.mjs";
+import { locateFrontmatterKey } from "./retired-fields.mjs";
+import { loadPackConfig } from "./pack-config.mjs";
+// The record accessors, which live apart so that a module the compilers load
+// can read a record without importing this one and closing a cycle (#243).
+// Re-exported because this is where callers have always addressed them.
+import { authoredFrontmatter, DERIVED_KEYS, isNoteRecord, noteFile } from "./index-records.mjs";
 
-        const slug = anchor[2].trim();
-        if (!slug) continue;
-        anchors.push({
-            slug,
-            name: anchor[1].trim(),
-            level: heading[1].length,
-            line: bodyLine + i,
-        });
-    }
-    return anchors;
-}
+export { authoredFrontmatter, DERIVED_KEYS, isNoteRecord, noteFile };
 
 /**
  * The address a wikilink writes to reach a note, or `null` when it has none.
  *
  * A wikilink target is an address: `being-aurochs` locally, or
  * `sohl-being-aurochs` from another package (`readQualifier` also accepts
- * `being/aurochs`, the same two fields with a different separator). Both forms
- * are already derivable from `type` and `shortcode`, which every record
- * carries — so this field adds no information. What it adds is the *rule*:
- * the lowercasing and the hyphen join live in one place, and a consumer that
- * reimplements them slightly differently gets a lookup that matches nothing and
- * says nothing about why. That is a real failure, not a hypothetical one — it
- * is precisely how a resolver keyed on a bare `type/shortcode` silently misses
- * every canonical `pkg-type-shortcode` entry.
+ * `being/aurochs`, the same two fields with a different separator). Both are
+ * *partial* addresses — the canonical one this field records is
+ * `sohl-sohl-being-aurochs`, and a target that omits the system is matched
+ * with that segment wildcarded. Every form is already derivable from `type` and
+ * `shortcode`, which every record carries, plus the system the type compiles
+ * into — so this field adds no information. What it adds is the *rule*: the
+ * lowercasing, the hyphen join and the system lookup live in one place, and a
+ * consumer that reimplements them slightly differently gets a lookup that
+ * matches nothing and says nothing about why. That is a real failure, not a
+ * hypothetical one — it is precisely how a resolver keyed on a bare
+ * `type/shortcode` silently misses every canonical
+ * `pkg-system-type-shortcode` entry.
  *
  * Derived by the same functions the link manifest and the site build use, so an
  * index cannot disagree with either about where a note lives.
@@ -181,8 +150,8 @@ export function collectAnchors(body, bodyLine = 1) {
  * @param {Record<string, any>} frontmatter - The note's parsed frontmatter.
  * @param {string} contentPackage - The package the tree compiles as.
  * @returns {{slug: string, canonical: string}|null} `slug` is what goes inside
- *   `[[…]]` within this package; `canonical` is the package-qualified key the
- *   manifest files the note under. `null` for a note with no type or no
+ *   `[[…]]` within this package; `canonical` is the fully qualified key the
+ *   manifest files the note under, carrying the package and the system as well. `null` for a note with no type or no
  *   shortcode, which has no address at all and is stated as such rather than
  *   left for every reader to rediscover.
  */
@@ -198,7 +167,12 @@ export function noteAddress(frontmatter, contentPackage) {
     }
     return {
         slug,
-        canonical: canonicalKey(contentPackage, frontmatter.type, frontmatter.shortcode),
+        canonical: canonicalKey(
+            contentPackage,
+            systemOf(frontmatter.type, KNOWN_DOCUMENT_SUBTYPE_MAPS),
+            frontmatter.type,
+            frontmatter.shortcode,
+        ),
     };
 }
 
@@ -355,35 +329,97 @@ function foundryEntries({ frontmatter, address, body, manifest }) {
 }
 
 /**
- * The `foundry` block for one manifest entry.
+ * The `foundry` block for one manifest entry, **keyed by the system that
+ * compiles it**.
+ *
+ * A note may declare more than one system — 2,497 of `harn-ensemble`'s carry
+ * both a `sohl:` and an `hm3:` block — and each compiles into its *own* Foundry
+ * document, of that system's document type, in that system's pack. One `uuid`
+ * on the record cannot name two documents, so it named whichever the single
+ * shipped map produced and said nothing about the other.
+ *
+ * Only `sohl` can appear today, because `KNOWN_DOCUMENT_SUBTYPE_MAPS` holds one
+ * map and #139 tracks the missing `hm3/` half. The shape is system-keyed now so
+ * that adding it is one more key rather than a second breaking change to an
+ * artifact consumers have already started reading.
+ *
+ * **Derived, and a sibling of the authored block rather than inside it.** The
+ * uuid could have been synthesized onto `sohl:`/`hm3:` themselves, but those are
+ * regions a note *authors*, and {@link DERIVED_KEYS} — which refuses a note that
+ * writes over derived data — reaches only the top level. A note authoring
+ * `sohl.uuid` would collide silently, which is the failure this index exists to
+ * stop rather than to add.
  *
  * @param {object|null} entry - A manifest entry.
- * @returns {object|null} `{ uuid?, anchors? }`, or null when it addresses nothing.
+ * @param {string} system - The system whose document this is.
+ * @returns {object|null} `{ [system]: { uuid?, anchors? } }`, or null when it
+ *   addresses nothing.
  */
-function foundryBlock(entry) {
+
+function foundryBlock(entry, system) {
     if (!entry) return null;
     const block = {};
     if (entry.uuid) block.uuid = entry.uuid;
     if (entry.anchors) block.anchors = entry.anchors;
-    return Object.keys(block).length ? block : null;
+    if (!Object.keys(block).length) return null;
+    return { [system || NO_SYSTEM]: block };
+}
+
+/**
+ * Refuse a note that authors a key the index derives, and say where.
+ *
+ * **Located, because every reader of the index is now a reporter of this.**
+ * Until #243 the only pass that built a record was the emitter, so aborting
+ * with a bare message was the whole story. Now the link check and the address
+ * diff read the index too, and a bare abort in one of them reports *nothing*
+ * about the tree — the one malformed note takes every other finding with it,
+ * and the reader is handed a stack instead of a line to open. So the error
+ * carries `file` and a `position`, and a pass that collects rather than throws
+ * can emit `file:line:column: error: …` like any other finding.
+ *
+ * **`package:` keeps its own words.** It is not a name collision but a *retired
+ * field* (#56), and the correction is to delete it, not to rename it — which is
+ * what {@link module:engine/note-package.assertNoDeclaredPackage} has always
+ * said, and had no caller to say it to. Deferring to it means one message for
+ * one mistake rather than two that contradict each other about the fix.
+ *
+ * @param {object} frontmatter - The note's parsed frontmatter.
+ * @param {string} relPath - The note's path within the tree, for the message.
+ * @param {string} [absPath] - The file, read only on the failing path to locate
+ *   the offending key.
+ * @param {string} [contentPackage] - The package this tree compiles as.
+ * @returns {void}
+ * @throws {Error} When the note authors a derived key. `file` and `position`
+ *   ride on the error.
+ */
+function assertNoDerivedKeys(frontmatter, relPath, absPath, contentPackage) {
+    for (const key of DERIVED_KEYS) {
+        if (!Object.hasOwn(frontmatter ?? {}, key)) continue;
+        if (key === "package") {
+            // Throws with its own wording, and its own position.
+            assertNoDeclaredPackage(frontmatter, { absPath, configured: contentPackage });
+        }
+        const err = new Error(
+            `\`${key}:\` is derived by the content index and cannot be ` +
+                `authored — rename the frontmatter field`,
+        );
+        err.file = relPath;
+        const position = absPath ? locateFrontmatterKey(absPath, key) : undefined;
+        if (position) err.position = position;
+        throw err;
+    }
 }
 
 export function buildIndexRecord({
     frontmatter,
     relPath,
+    absPath,
     contentPackage,
     body,
     bodyLine,
     manifest,
 }) {
-    for (const key of DERIVED_KEYS) {
-        if (Object.hasOwn(frontmatter ?? {}, key)) {
-            throw new Error(
-                `${relPath}: \`${key}:\` is derived by the content index and ` +
-                    `cannot be authored — rename the frontmatter field`,
-            );
-        }
-    }
+    assertNoDerivedKeys(frontmatter, relPath, absPath, contentPackage);
 
     const posix = relPath.split(path.sep).join("/");
     const folder = posix.includes("/") ? posix.slice(0, posix.lastIndexOf("/")) : "";
@@ -405,7 +441,10 @@ export function buildIndexRecord({
                 ...a,
                 link: address ? `${address.slug}#${a.slug}` : null,
             })),
-            foundry: foundryBlock(entries?.own),
+            foundry: foundryBlock(
+                entries?.own,
+                systemOf(frontmatter?.type, KNOWN_DOCUMENT_SUBTYPE_MAPS),
+            ),
             // Forward link to the note's documentation journal, which is its
             // own record. Named rather than nested, because the journal is a
             // separate document with its own address — see `buildDocRecord`.
@@ -481,30 +520,58 @@ function buildDocRecord({ frontmatter, address, entry, file, contentPackage, anc
             // the forward link on that record, so either end reaches the other.
             documents: address.canonical,
             anchors,
-            foundry: foundryBlock(entry),
+            foundry: foundryBlock(entry), // a journal: no system key
             file,
         })
     );
 }
 
-export function collectContentIndex(contentBase, { contentPackage, skipDirectories, manifest }) {
+export function collectContentIndex(
+    contentBase,
+    { contentPackage, skipDirectories, manifest, problems },
+) {
     const records = [];
-    const walkOpts = skipDirectories ? { skipDirectories } : {};
+    // Passed through rather than defaulted away: an absent scope is the
+    // caller's omission, and `walkMarkdownTree` says so (#243).
+    const walkOpts = { skipDirectories };
 
     for (const { frontmatter, body, bodyLine, absPath } of walkMarkdownTree(
         contentBase,
         walkOpts,
     )) {
         const fm = frontmatter ?? {};
+        // The id the note's document is filed under (#270), resolved before
+        // the record is built so the index publishes the address *and* the id
+        // that address derives.
+        resolveNoteId(fm, { pkg: contentPackage });
         const relPath = path.relative(contentBase, absPath);
-        const record = buildIndexRecord({
-            frontmatter: fm,
-            relPath,
-            contentPackage,
-            body,
-            bodyLine,
-            manifest,
-        });
+        let record;
+        try {
+            record = buildIndexRecord({
+                frontmatter: fm,
+                relPath,
+                absPath,
+                contentPackage,
+                body,
+                bodyLine,
+                manifest,
+            });
+        } catch (err) {
+            // No `problems` array means the caller wants the old contract: a
+            // note that cannot be recorded fails the derivation outright, which
+            // is right for the *emitter* — an index quietly missing a note
+            // would state that the note does not exist.
+            if (!problems) throw err;
+            // A reader, by contrast, reports it and carries on: one malformed
+            // note must not take every other finding in the tree with it.
+            problems.push({
+                file: absPath,
+                ...(err.position ?? {}),
+                severity: "error",
+                message: String(err.message),
+            });
+            continue;
+        }
         records.push(record);
 
         // An item note is two documents, so it is two records (#239).
@@ -562,6 +629,43 @@ export function serializeContentIndex(records) {
     return `${records.map((r) => JSON.stringify(r)).join("\n")}\n`;
 }
 
+/**
+ * The index records for a content tree, without writing anything.
+ *
+ * The half of {@link emitContentIndex} that derives rather than emits, so a
+ * pass that needs the corpus in memory — a SQL content table, the link check,
+ * and in time every reader #243 converts — builds it the same way the artifact
+ * is built, rather than by walking and parsing again with its own idea of the
+ * scope.
+ *
+ * @param {object} [opts]
+ * @param {string} [opts.contentBase] - The tree, defaulting to the configured one.
+ * @param {object} [opts.config] - Resolved configuration, defaulting to ambient.
+ * @param {readonly string[]} [opts.skipDirectories] - The walk's scope, for a
+ *   caller that resolved one of its own; defaults to the resolved
+ *   configuration's. Stated separately from `config` because a caller that was
+ *   *handed* a scope must be able to pass it on rather than have it silently
+ *   replaced by the one its configuration happens to carry (#243).
+ * @param {object[]} [opts.problems] - Supplied by a **reader**: a note that
+ *   cannot be recorded is pushed here as a diagnostic and skipped, instead of
+ *   aborting the derivation. Omitted, the note throws — which is the contract
+ *   the emitter needs, since an index missing a note asserts that it does not
+ *   exist.
+ * @returns {object[]} One record per note, plus one per documentation entry.
+ */
+export function indexRecordsFor({ contentBase, config, skipDirectories, problems } = {}) {
+    const resolved = config ?? loadPackConfig();
+    const tree = contentBase ?? resolved.paths.content;
+    if (!fs.existsSync(tree)) throw new Error(`no content tree at ${tree}`);
+    return collectContentIndex(tree, {
+        contentPackage: resolved.contentPackage,
+        skipDirectories: skipDirectories ?? resolved.skipDirectories,
+        // Only the identities a UUID is a function of — see emitContentIndex.
+        manifest: foundryIdentities(resolved),
+        problems,
+    });
+}
+
 /**
  * Emit this package's content index.
  *
@@ -597,13 +701,7 @@ export function emitContentIndex({ contentBase, outDir, config } = {}) {
     // publishes pages is no part of an address, and depending on it would make
     // the index refuse to build for a configuration that is perfectly able to
     // state one.
-    const manifest = foundryIdentities(resolved);
-
-    const records = collectContentIndex(tree, {
-        contentPackage,
-        skipDirectories: resolved.skipDirectories,
-        manifest,
-    });
+    const records = indexRecordsFor({ contentBase: tree, config: resolved });
     if (records.length === 0) {
         throw new Error(
             `${tree} yielded no notes, so the index would state that this ` +
@@ -612,7 +710,7 @@ export function emitContentIndex({ contentBase, outDir, config } = {}) {
     }
 
     const text = serializeContentIndex(records);
-    const file = path.join(dir, `${contentPackage}.jsonl`);
+    const file = path.join(dir, metadataFileName(contentPackage));
     fs.mkdirSync(dir, { recursive: true });
     fs.writeFileSync(file, text);