@heroiclands/package-build 17.1.0 → 18.0.0

package/engine/schema-check.mjs

@@ -39,7 +39,7 @@
  *
  * **The rest of the emission is only observable.** A compiler writes keys of
  * its own alongside the declared fields — `shortcode`, `actionDefs`, `notes`,
- * `docHtml`, and since #126 `archetype` — and those appear in neither set
+ * `docHtml`, and since #126 `templatePriority` — and those appear in neither set
  * {@link compareFields} compares, so nothing compared them at all (#155). They
  * cannot be listed here without the list going stale the next time a compiler
  * grows a key, so they are read off the `system` object the compiler produced:
@@ -357,7 +357,7 @@ function undeclaredEmittedPaths(data, declared, enumerated, prefix = "") {
  * *observed*. {@link compareFields} reads the `itemBuilders` declarations and
  * {@link checkAuthoredSystemData} reads a note's `<system>.system`; between them
  * they miss every key a compiler writes on its own initiative, which is not a
- * residue — it is `shortcode`, `actionDefs`, `notes`, `docHtml` and `archetype`.
+ * residue — it is `shortcode`, `actionDefs`, `notes`, `docHtml` and `templatePriority`.
  *
  * **The keys come from the object the compiler built, after a JSON round trip.**
  * That is exactly what the pack file receives, so a key whose value is
@@ -455,12 +455,25 @@ export function compareEmittedSystem({
  * stamps no system at all, and a system that has not adopted the artifact yet
  * is simply unchecked. Neither is an error, and the caller says which it was.
  *
+ * **A build may have more than one system, and then the caller names it (#139).**
+ * `stats.systemId` is the package-wide answer, and a repository shipping content
+ * for two systems has no package-wide answer — it is deliberately `null` there,
+ * because a module feeding both `sohl` and `hm3` targets neither. Left at that,
+ * every schema check in such a build would be skipped in silence, which is the
+ * state #60 exists to remove: the five type names the two systems *share* are
+ * exactly the ones a wrong-system emission hides in. So a pass supplies the
+ * system its pack declares, and the version comes from that system's own
+ * `systems:` entry rather than from a package-wide stamp.
+ *
  * @param {object} config - The resolved build configuration.
+ * @param {string|null} [system] - The system whose schema is wanted. Defaults
+ *   to the package-wide `stats.systemId`, which is every single-system build
+ *   and the behaviour this always had.
  * @returns {{artifact: SchemaArtifact, source: string}|null} The schema and
  *   where it was read from.
  */
-export function resolveSchemaArtifact(config) {
-    const systemId = config?.stats?.systemId;
+export function resolveSchemaArtifact(config, system = undefined) {
+    const systemId = system === undefined ? config?.stats?.systemId : system;
     if (!systemId) return null;
 
     const read = (file) => ({
@@ -474,7 +487,13 @@ export function resolveSchemaArtifact(config) {
         return fs.existsSync(own) ? read(own) : null;
     }
 
-    const version = config?.stats?.systemVersion;
+    // The version this build compiles *against*, for this system. A `systems:`
+    // entry is the per-system statement and wins; `stats.systemVersion` is the
+    // package-wide one and answers only for the package-wide system, which is
+    // what keeps a module still deriving from `relationships.systems` working.
+    const version =
+        config?.systems?.[systemId]?.compatibility?.verified ??
+        (systemId === config?.stats?.systemId ? config?.stats?.systemVersion : null);
     if (!version) return null;
     const cached = cachedSchemaPath(config, systemId, version);
     return fs.existsSync(cached) ? read(cached) : null;
@@ -556,22 +575,29 @@ export function emittedUndeclaredMessage(finding) {
     );
 }
 
-/** One resolved schema artifact per resolved configuration. */
+/** One resolved schema artifact per configuration **and system**. */
 const artifacts = new WeakMap();
 
 /**
- * {@link resolveSchemaArtifact}, read once per configuration.
+ * {@link resolveSchemaArtifact}, read once per configuration and system.
  *
- * The per-note check below runs thousands of times in a build and the artifact
+ * The per-note check below runs thousands of times in a build and an artifact
  * never changes inside one, so reading and parsing it per note would be a
  * megabyte of JSON per hundred documents for an answer that is already known.
+ * Keyed by system as well as by configuration since #139: a build with two
+ * systems has two artifacts, and caching one of them under the configuration
+ * alone would hand every pass whichever system asked first.
  *
  * @param {object} config - The resolved build configuration.
+ * @param {string|null} [system] - The system whose schema is wanted.
  * @returns {{artifact: SchemaArtifact, source: string}|null} The schema.
  */
-function schemaFor(config) {
-    if (!artifacts.has(config)) artifacts.set(config, resolveSchemaArtifact(config));
-    return artifacts.get(config);
+function schemaFor(config, system = undefined) {
+    let bySystem = artifacts.get(config);
+    if (!bySystem) artifacts.set(config, (bySystem = new Map()));
+    const key = system === undefined ? null : system;
+    if (!bySystem.has(key)) bySystem.set(key, resolveSchemaArtifact(config, system));
+    return bySystem.get(key);
 }
 
 /**
@@ -600,18 +626,21 @@ function schemaFor(config) {
  * @param {string} opts.block - The system block to read, e.g. `"sohl"`.
  * @param {string} opts.documentType - `Item`, `Actor`, …
  * @param {string} opts.subType - The document subtype the note compiles into.
+ * @param {string} [opts.system] - The system whose published schema to read,
+ *   where a build has more than one (#139). Defaults to the package-wide
+ *   `stats.systemId`.
  * @param {object} [opts.config] - The resolved build configuration.
  * @returns {{path: string, message: string}[]} One finding per undeclared path,
  *   shallowest-first.
  */
 export function checkAuthoredSystemData(
     fm,
-    { block, documentType, subType, config = loadPackConfig() },
+    { block, documentType, subType, system = undefined, config = loadPackConfig() },
 ) {
     const data = systemData(fm, block);
     if (!Object.keys(data).length) return [];
 
-    const schema = schemaFor(config);
+    const schema = schemaFor(config, system);
     if (!schema) return [];
     const declared = declaredFields(schema.artifact, documentType, subType);
     if (!declared) return [];
@@ -651,18 +680,30 @@ export function checkAuthoredSystemData(
  * @param {string} opts.type - The note's content type, for the message.
  * @param {readonly {to?: string}[]} [opts.fields] - The type's field
  *   declaration, which decides each finding's origin.
+ * @param {string} [opts.system] - The system whose published schema to read,
+ *   where a build has more than one (#139). Defaults to the package-wide
+ *   `stats.systemId`.
  * @param {object} [opts.config] - The resolved build configuration.
  * @returns {(EmissionFinding & {message: string})[]} One per undeclared path.
  */
 export function checkEmittedSystemData(
-    system,
-    { fm, block, documentType, subType, type, fields, config = loadPackConfig() },
+    emitted,
+    {
+        fm,
+        block,
+        documentType,
+        subType,
+        type,
+        fields,
+        system = undefined,
+        config = loadPackConfig(),
+    },
 ) {
-    const schema = schemaFor(config);
+    const schema = schemaFor(config, system);
     if (!schema) return [];
 
     return compareEmittedSystem({
-        system,
+        system: emitted,
         artifact: schema.artifact,
         documentType,
         subtype: subType,

package/engine/site-build.mjs

@@ -55,11 +55,14 @@ import { protectCode } from "./code-fences.mjs";
 import { expandContentTables } from "./content-tables.mjs";
 import { buildSiteIndex, wikiContext } from "./site-index.mjs";
 import { frontmatterWikilinks, resolveWebWikilinks } from "./web-wikilinks.mjs";
-import { loadForeignManifests, manifestsComplete } from "./kb-manifest.mjs";
-import { formatUnaddressableFinding, unaddressableForeignPackages } from "./foreign-manifests.mjs";
+import { loadForeignIndexes } from "./metadata-index.mjs";
+import { formatUnaddressableFinding, unaddressableForeignPackages } from "./metadata-index.mjs";
 import { deriveBeingInfo, isBeing } from "../sohl/being-info.mjs";
 import { loadPackConfig } from "./pack-config.mjs";
 import { searchableFrontmatter } from "./note-package.mjs";
+// The corpus, from the one pass that derives it (#243).
+import { indexRecordsFor } from "./content-index.mjs";
+import { isNoteRecord, noteFile } from "./index-records.mjs";
 import {
     checkHomepageCount,
     homepageDestination,
@@ -102,6 +105,50 @@ export function walkSiteTree(dir, skip = []) {
     return out;
 }
 
+/**
+ * The content-tree files this build publishes from, in the order it emits them.
+ *
+ * **The corpus comes from the content index** (#243) — the same derivation the
+ * packs are compiled from — so the site and the packs cannot disagree about
+ * which files are the content. The note is still read for its `{fm, body}`: the
+ * index carries no note text, and a page *is* its text.
+ *
+ * **The order changes, and that is the point of stating it here.** The walk this
+ * replaces yielded directory order, and this module kept it deliberately —
+ * "a site's emitted pages should not reorder for no reason". Records are in
+ * content-path order, which is the same set in a different sequence. Nothing
+ * downstream depends on it any more: the first-writer-wins fallbacks that made
+ * order load-bearing went with the bare `[[Name]]` form (#180), each page is
+ * emitted to its own file at an address derived from its frontmatter, and the
+ * one place order could still show — a section's page list — is sorted by the
+ * theme. Verified rather than argued: over `sohl`'s tree the emitted mount is
+ * byte-identical, all 1,749 files.
+ *
+ * A **content-path** order is also the better of the two. Directory-read order
+ * is a fact about the filesystem, not about the content, so it can differ
+ * between two checkouts of one tree; this order cannot.
+ *
+ * `collectTreePages` is deliberately **not** converted: it walks an auxiliary
+ * tree (`site.trees`, the developer docs), which is not the content tree and
+ * appears in no record.
+ *
+ * @param {string} contentBase - Root of the content tree.
+ * @param {object} ctx - The build context. `ctx.records` is the corpus when the
+ *   caller already derived it — the site build derives one and hands it to both
+ *   collectors, so the two cannot disagree.
+ * @returns {string[]} Absolute paths, in emission order.
+ */
+function siteCorpusFiles(contentBase, ctx) {
+    const records =
+        ctx.records ??
+        indexRecordsFor({
+            contentBase,
+            config: ctx.config,
+            skipDirectories: ctx.skipDirectories,
+        });
+    return records.filter(isNoteRecord).map((record) => noteFile(contentBase, record));
+}
+
 /**
  * Reads a note, returning `null` for one that cannot be parsed.
  *
@@ -145,7 +192,7 @@ export function collectContentPages(contentBase, ctx) {
     const addressFindings = [];
     const fmLinkFindings = [];
 
-    for (const file of walkSiteTree(contentBase, ctx.skipDirectories)) {
+    for (const file of siteCorpusFiles(contentBase, ctx)) {
         const note = readNote(file);
         if (!note) continue;
         const { fm, body } = note;
@@ -304,7 +351,7 @@ export function collectTreePages(tree, ctx) {
 export function collectHomepages(contentBase, ctx) {
     const pages = [];
     const addressFindings = [];
-    for (const file of walkSiteTree(contentBase, ctx.skipDirectories)) {
+    for (const file of siteCorpusFiles(contentBase, ctx)) {
         const note = readNote(file);
         if (!note || !isHomepage(note.fm)) continue;
         try {
@@ -380,10 +427,10 @@ export function writeHomepages(outRoot, pages, config) {
  * @param {object[]} pages - Every page, from both walks.
  * @param {object} findings - `{ addressFindings, fmLinkFindings }` from
  *   collection.
- * @param {object} options - `{ manifestDir }`.
+ * @param {object} options - `{ config }`.
  * @returns {object} The gate results and, when they pass, the built index.
  */
-export function siteGates(pages, findings, { manifestDir }) {
+export function siteGates(pages, findings, { config }) {
     const out = {
         // Always empty here: the homepage count is decided in `buildSite`
         // before the content walk, and a failing count returns without ever
@@ -406,7 +453,7 @@ export function siteGates(pages, findings, { manifestDir }) {
     // configured list instead silently discarded the manifest of any package
     // the list named but the tree did not contain.
     const localPackages = new Set(content.map((p) => p.pkg));
-    const foreign = loadForeignManifests(manifestDir, localPackages);
+    const foreign = loadForeignIndexes(config, localPackages);
     out.foreign = foreign;
     if (foreign.stale.length) {
         out.staleManifests = foreign.stale;
@@ -416,7 +463,6 @@ export function siteGates(pages, findings, { manifestDir }) {
     out.unaddressable = unaddressableForeignPackages(foreign.index);
     if (out.unaddressable.length) return out;
 
-    out.manifests = manifestsComplete(localPackages, foreign.packages);
     const index = buildSiteIndex(pages, { foreignIndex: foreign.index });
     out.conflicts = index.conflicts;
     if (out.conflicts.length) return out;
@@ -658,6 +704,7 @@ export function renderPages(pages, options) {
         readmeSections,
         decorate,
         linkable = (d) => Boolean(d.fm.shortcode),
+        sqlTables,
     } = options;
 
     const tableErrors = [];
@@ -692,6 +739,10 @@ export function renderPages(pages, options) {
                 docs: universe.get(page.pkg) ?? [],
                 linkable,
                 source: src,
+                // Prepared before this render began — DuckDB is async and this
+                // is not. Keyed by the note's own file, absolute here as in
+                // every other pass, so the three cannot disagree about a note.
+                sqlTables: sqlTables?.get(page.file),
                 self: {
                     fm: searchableFrontmatter(page.fm, page.pkg),
                     path: page.relPath,
@@ -898,7 +949,7 @@ export function resolveOutputRoot(rootDir, out) {
  * @returns {{gates: object, stats: object|null, tableErrors: object[],
  *   wikiErrors: object[], manifests: object|null}}
  */
-export function buildSite({ config, outRoot } = {}) {
+export function buildSite({ config, outRoot, sqlTables } = {}) {
     const resolved = config ?? loadPackConfig();
     const site = resolved.site;
     const scheme = resolved.publish.address;
@@ -940,6 +991,15 @@ export function buildSite({ config, outRoot } = {}) {
         // retired, so this is the only source of it (#56).
         contentPackage: resolved.contentPackage,
         skipDirectories: resolved.skipDirectories,
+        config: resolved,
+        // The corpus, derived once for this build and handed to both
+        // collectors — the homepage pass and the content pass read one answer
+        // about which files the content is, rather than walking twice (#243).
+        records: indexRecordsFor({
+            contentBase: resolved.paths.content,
+            config: resolved,
+            skipDirectories: resolved.skipDirectories,
+        }),
         // Where the package is served, which is where an addressed page
         // publishes: an address is `(type, shortcode)`, a package-wide
         // identity that takes no content mount (#181).
@@ -1041,7 +1101,7 @@ export function buildSite({ config, outRoot } = {}) {
     const gates = siteGates(
         [...pages, ...homepageEntries],
         { ...content, fmLinkFindings },
-        { manifestDir: resolved.paths.manifests },
+        { config: resolved },
     );
     if (gatesFailed(gates)) {
         return {
@@ -1049,7 +1109,6 @@ export function buildSite({ config, outRoot } = {}) {
             stats: null,
             tableErrors: [],
             wikiErrors: [],
-            manifests: gates.manifests,
         };
     }
 
@@ -1060,6 +1119,7 @@ export function buildSite({ config, outRoot } = {}) {
 
     const rendered = renderPages(pages, {
         outRoot: out,
+        sqlTables,
         index: gates.index,
         foreign: gates.foreign,
         universe: tableUniverse(pages),
@@ -1088,7 +1148,6 @@ export function buildSite({ config, outRoot } = {}) {
 
     return {
         gates,
-        manifests: gates.manifests,
         tableErrors: rendered.tableErrors,
         wikiErrors: rendered.wikiErrors,
         stats: {

package/engine/site-index.mjs

@@ -30,8 +30,11 @@
  * **Two key spaces, one map**, and both are addresses. `section/slug` and
  * `type/shortcode` are unique by construction, so they always resolve.
  * `type/shortcode` is the authored form; the canonical
- * `package-type-shortcode` is set alongside it, which is what a cross-package
- * link and every merged foreign entry use (#1499).
+ * `package-system-type-shortcode` is set alongside it, which is what a
+ * cross-package link and every merged foreign entry use (#1499, #59). A
+ * cross-package target states its package and usually not its system, so it is
+ * matched by the segments it supplies rather than fetched by an exact key —
+ * one hit resolves, and anything else is a finding.
  *
  * **A page's *name* is not a key** (#180). It was, as one of a set of
  * collision-aware fallbacks a bare `[[Name]]` was looked up in — which is what
@@ -48,7 +51,9 @@
 
 import path from "node:path";
 
-import { canonicalKey, readCanonicalKey } from "./kb-manifest.mjs";
+import { canonicalKey, readCanonicalKey } from "./content-address.mjs";
+import { systemOf } from "./document-subtypes.mjs";
+import { KNOWN_DOCUMENT_SUBTYPE_MAPS } from "./note-claims.mjs";
 import { hasDocEntry } from "./item-docs.mjs";
 import { contentPackage } from "./content-package.mjs";
 // The declared tag vocabulary (#172), which is where `draft` is stated.
@@ -158,7 +163,7 @@ function mergeForeign(index, foreignIndex) {
  *   each already knowing its own `url`.
  * @param {object} [options] - Cross-package inputs.
  * @param {Map<string, {package: string, type?: string}>} [options.foreignIndex]
- *   The merged index from `loadForeignManifests`. Omit when the build publishes
+ *   The merged index from `loadForeignIndexes`. Omit when the build publishes
  *   no cross-package links.
  * @returns {SiteIndex} The index, and what could not be addressed unambiguously.
  */
@@ -208,8 +213,8 @@ export function buildSiteIndex(entries, { foreignIndex = new Map() } = {}) {
     }
 
     // Merged *before* the local type-scoped pass below, so a local page always
-    // ends up owning its own canonical `package-type-shortcode` address: the
-    // local write lands last and wins. `loadForeignManifests` already excludes
+    // ends up owning its own canonical `package-system-type-shortcode` address: the
+    // local write lands last and wins. `loadForeignIndexes` already excludes
     // the local packages, so a manifest should never carry one — this is what
     // makes that a belt-and-braces rather than the only thing standing between
     // a stale vendored manifest and a shadowed local page.
@@ -241,7 +246,15 @@ export function buildSiteIndex(entries, { foreignIndex = new Map() } = {}) {
             // The page's package is the configured one — the site collection
             // resolves it and records it as `pkg`. Never read out of
             // frontmatter: `package:` is retired (#56).
-            index.set(canonicalKey(e.pkg ?? ownPackage, type, shortcode), value);
+            index.set(
+                canonicalKey(
+                    e.pkg ?? ownPackage,
+                    systemOf(type, KNOWN_DOCUMENT_SUBTYPE_MAPS),
+                    type,
+                    shortcode,
+                ),
+                value,
+            );
             if (e.pkg) packages.add(e.pkg);
             // In Foundry an item and its documentation are two documents, so
             // `skill/wpnc` and `docskill/wpnc` are two UUIDs (#1362). Here the