@heroiclands/package-build 17.1.0 → 18.0.0

package/engine/foreign-catalog.mjs

@@ -47,6 +47,15 @@ import { extractPack } from "@foundryvtt/foundryvtt-cli";
 
 import log from "loglevel";
 
+import {
+    metadataRelationships,
+    metadataCacheDir,
+    metadataFileName,
+    newestVersionDir,
+    isComplete as metadataIsComplete,
+    markComplete as markMetadataComplete,
+} from "./metadata-index.mjs";
+
 /** Written once a fetch completes, so a half-finished cache is never used. */
 const STAMP = ".complete";
 
@@ -426,6 +435,185 @@ export async function fetchCatalogFromPath(config, rel, source) {
     }
 }
 
+/**
+ * Fetch one dependency's published content index (#239).
+ *
+ * **The chain is entirely declared.** The relationship names the dependency's
+ * manifest, the manifest advertises `flags.metadataUrl`, and that URL is the
+ * index — so nothing here holds an address of its own, and a dependency that
+ * moves its release assets does not break its consumers.
+ *
+ * Pinned by the same rule as the catalogue: `compatibility.verified` is the
+ * version this repository was built against, so a floating `releases/latest`
+ * URL is rewritten to it. A consumer resolving addresses against whatever the
+ * dependency published this morning is not reproducible.
+ *
+ * Idempotent: a complete cache for the resolved version is left alone.
+ *
+ * @param {object} config - The resolved build configuration.
+ * @param {{id: string, manifest: string, verified?: string}} rel - The declared
+ *   relationship.
+ * @returns {Promise<string>} The cached index file.
+ */
+export async function fetchMetadata(config, rel) {
+    const { url, pinned } = pinnedManifestUrl(rel.manifest, rel.verified);
+    const manifest = await fetchManifest(url);
+    const version = manifest.version;
+    if (!version) {
+        throw new Error(`${rel.id}: its manifest declares no \`version\``);
+    }
+    if (!pinned && rel.verified && version !== rel.verified) {
+        throw new Error(
+            `${rel.id}: declares \`compatibility.verified: ${rel.verified}\` but ` +
+                `${url} offers ${version}. Building against a moving target is ` +
+                `not reproducible — update \`verified\`, or point \`manifest\` ` +
+                `at a pinned release.`,
+        );
+    }
+
+    const dir = metadataCacheDir(config, rel.id, version);
+    const indexUrl = manifest.flags?.metadataUrl;
+    if (!indexUrl) {
+        throw new Error(
+            `${rel.id}@${version}: its manifest advertises no ` +
+                `\`flags.metadataUrl\`, so it publishes no content index and ` +
+                `nothing can link into it. It needs a release built with ` +
+                `package-build 18 or later.`,
+        );
+    }
+
+    if (metadataIsComplete(dir)) {
+        log.info(`${rel.id}@${version}: index already cached`);
+        return path.join(dir, path.basename(new URL(indexUrl).pathname));
+    }
+
+    // Rebuild from empty: a previous run may have died partway, and a stale
+    // half-written index is worse than none.
+    fs.rmSync(dir, { recursive: true, force: true });
+    fs.mkdirSync(dir, { recursive: true });
+
+    log.info(`${rel.id}@${version}: downloading ${indexUrl}`);
+    const res = await fetch(indexUrl, { redirect: "follow" });
+    if (!res.ok) {
+        throw new Error(
+            `${rel.id}@${version}: could not download its content index at ` +
+                `${indexUrl}: HTTP ${res.status} ${res.statusText}`,
+        );
+    }
+    const file = path.join(dir, path.basename(new URL(indexUrl).pathname));
+    fs.writeFileSync(file, Buffer.from(await res.arrayBuffer()));
+    markMetadataComplete(dir);
+    return file;
+}
+
+/**
+ * Fill the index cache from a locally built artifact rather than a release.
+ *
+ * The counterpart of {@link fetchCatalogFromPath}, and the same escape hatch
+ * for the same reason: two packages being changed together cannot each wait for
+ * the other to ship. The index is looked for beside the manifest — which is
+ * where a build leaves it and where the release publishes it — so a package
+ * directory and an unpacked zip are both usable as-is.
+ *
+ * @param {object} config - The resolved build configuration.
+ * @param {{id: string}} rel - The declared relationship.
+ * @param {string} source - Path to the artifact or its directory.
+ * @returns {Promise<string>} The cached index file.
+ */
+export async function fetchMetadataFromPath(config, rel, source) {
+    if (!fs.existsSync(source)) {
+        throw new Error(`${rel.id}: nothing at ${source}`);
+    }
+
+    const staging = fs.mkdtempSync(path.join(os.tmpdir(), `content-build-meta-${rel.id}-`));
+    try {
+        let root = source;
+        if (!fs.statSync(source).isDirectory()) {
+            writeZipEntries(unzipSync(new Uint8Array(fs.readFileSync(source))), staging);
+            root = staging;
+        }
+
+        const manifest = readLocalManifest(root);
+        if (!manifest) {
+            throw new Error(
+                `${rel.id}: ${source} holds no system.json or module.json, so ` +
+                    `its version cannot be read`,
+            );
+        }
+        if (manifest.id && manifest.id !== rel.id) {
+            throw new Error(`${rel.id}: ${source} is package "${manifest.id}", not "${rel.id}"`);
+        }
+        const version = manifest.version;
+        if (!version) {
+            throw new Error(`${rel.id}: ${source} declares no \`version\``);
+        }
+
+        // Named by the manifest where it advertises one, so a local artifact
+        // and a released one are cached under the same name; falling back to
+        // the id covers a build whose manifest predates the flag.
+        const name =
+            manifest.flags?.metadataUrl ?
+                path.basename(new URL(manifest.flags.metadataUrl).pathname)
+            :   metadataFileName(rel.id);
+        const found = findLocalIndex(root, name);
+        if (!found) {
+            throw new Error(
+                `${rel.id}: ${source} holds no ${name}, so it publishes no ` +
+                    `content index. Build it before fetching from it.`,
+            );
+        }
+
+        const dir = metadataCacheDir(config, rel.id, version);
+        fs.rmSync(dir, { recursive: true, force: true });
+        fs.mkdirSync(dir, { recursive: true });
+        const file = path.join(dir, name);
+        fs.copyFileSync(found, file);
+        markMetadataComplete(dir);
+        log.info(`${rel.id}@${version}: index cached from ${source}`);
+        return file;
+    } finally {
+        fs.rmSync(staging, { recursive: true, force: true });
+    }
+}
+
+/**
+ * Locate an index file in an unpacked artifact.
+ *
+ * Foundry archives are inconsistent about whether they nest their contents
+ * under a top-level directory, so try the root and then one level in — the same
+ * allowance {@link resolvePackPath} makes for packs.
+ *
+ * @param {string} root - The unpacked package root.
+ * @param {string} name - The index file name.
+ * @returns {string|null} The path, or null when absent.
+ */
+function findLocalIndex(root, name) {
+    const direct = path.join(root, name);
+    if (fs.existsSync(direct)) return direct;
+    for (const entry of fs.readdirSync(root, { withFileTypes: true })) {
+        if (!entry.isDirectory()) continue;
+        const nested = path.join(root, entry.name, name);
+        if (fs.existsSync(nested)) return nested;
+    }
+    return null;
+}
+
+/**
+ * Fetch every declared dependency's content index.
+ *
+ * A wider set than {@link fetchAllCatalogs}: an index is fetched for *every*
+ * dependency, a catalogue only for those declaring `itemCatalog: true`. See
+ * {@link metadataRelationships} for why the two sets differ.
+ *
+ * @param {object} config - The resolved build configuration.
+ * @returns {Promise<number>} How many indexes were fetched.
+ */
+export async function fetchAllMetadata(config) {
+    const rels = metadataRelationships(config);
+    for (const rel of rels) await fetchMetadata(config, rel);
+    return rels.length;
+}
+
 /**
  * Fetch every opted-in dependency. The `deps fetch` command.
  *
@@ -470,10 +658,13 @@ export function foreignItemCatalogDirs(config) {
                     `fetched. Run \`content-build deps fetch\` first.`,
             );
         }
-        // Newest last wins if several versions are cached; a fetch always
-        // writes the currently declared one, so that is the one to use.
-        cached.sort();
-        const items = itemsDir(cached[cached.length - 1]);
+        // Newest wins if several versions are cached; a fetch always writes
+        // the currently declared one, so that is the one to use. The
+        // comparison is the content-index cache's, shared rather than
+        // rewritten: a plain string sort would put `0.8.10` before `0.8.2` and
+        // silently resolve every embedded item against the older catalogue
+        // (#272).
+        const items = itemsDir(newestVersionDir(cached));
         for (const name of fs.readdirSync(items)) {
             dirs.push(path.join(items, name));
         }

package/engine/{manifest-emit.mjs → foundry-entries.mjs}

@@ -14,7 +14,7 @@
 /**
  * Emitting this package's cross-package link manifest (#58).
  *
- * `engine/kb-manifest.mjs` owns the *format* — what an entry may say, how a
+ * `engine/content-address.mjs` owns the address *grammar* — how a key is
  * version is read, how a foreign file resolves. This module owns the *pass*:
  * walking a content tree and deriving, for every note it publishes, the
  * addresses that entry states. The two halves were split across the format
@@ -30,9 +30,12 @@
  * a fact it has to be told (#1465).
  *
  * **An entry's `path` is derivable from the key it is filed under** (#181).
- * `sohl-affliction-aconite` publishes at `affliction-aconite/`, because a page's
- * URL *is* its address; nothing in it comes from a display name, so a rename
- * moves no URL and no uniqueness check stands between the two. Every entry is
+ * `sohl-sohl-affliction-aconite` publishes at `affliction-aconite/` — the key
+ * with its package and system segments dropped — because a page's URL *is* its
+ * address; nothing in it comes from a display name, so a rename moves no URL and
+ * no uniqueness check stands between the two. The system segment goes with the
+ * package because a note publishes one page however many systems' documents it
+ * compiles into (#59). Every entry is
  * derivable that way since #204 retired the section landing, which was the one
  * that was not. The field is still written rather than left for a consumer to
  * compute, because an absent `path` already means something else entirely (a
@@ -55,10 +58,16 @@
 import fs from "node:fs";
 import path from "node:path";
 
-import { packageAddress } from "./content-address.mjs";
-import { canonicalKey, writeManifests } from "./kb-manifest.mjs";
+import { canonicalKey, packageAddress } from "./content-address.mjs";
+import { NO_SYSTEM, systemOf } from "./document-subtypes.mjs";
+import {
+    KNOWN_DOCUMENT_SUBTYPE_MAPS,
+    NEVER_PACKED_TYPES,
+    DERIVED_PACKED_TYPES,
+} from "./note-claims.mjs";
 import { walkMarkdownTree } from "./helpers.mjs";
-import { compendiumUuid, packForType, pageUuid } from "./ids.mjs";
+import { resolveNoteId } from "./note-ids.mjs";
+import { compendiumUuid, currentType, packForType, pageUuid } from "./ids.mjs";
 import { hasDocEntry, itemDocEntryId } from "./item-docs.mjs";
 import { isHomepage } from "./homepage.mjs";
 import { assertNoDeclaredPackage } from "./note-package.mjs";
@@ -66,6 +75,7 @@ import {
     assertNoAliasesField,
     assertNoDraftField,
     assertNoSectionField,
+    assertNoTraitsField,
 } from "./retired-fields.mjs";
 import { journalPageId, splitPages } from "./journals.mjs";
 import { routerFor } from "./pack-router.mjs";
@@ -100,7 +110,7 @@ export const LEAD_ANCHOR = "$lead";
 export function anchorsOf(entryUuid, entryId, body, name) {
     const anchors = {};
     splitPages(body, name).forEach((page, index) => {
-        const uuid = pageUuid(entryUuid, journalPageId(entryId, page, index));
+        const uuid = pageUuid(entryUuid, journalPageId(entryId, page));
         if (index === 0) anchors[LEAD_ANCHOR] = uuid;
         if (page.anchorSlug) anchors[page.anchorSlug] = uuid;
     });
@@ -128,7 +138,12 @@ export function anchorsOf(entryUuid, entryId, body, name) {
  */
 export function entriesForNote(fm, name, address, body, ctx) {
     const { contentPackage, foundryPackageId, packRouter } = ctx;
-    const key = canonicalKey(contentPackage, fm.type, fm.shortcode);
+    const key = canonicalKey(
+        contentPackage,
+        systemOf(fm.type, KNOWN_DOCUMENT_SUBTYPE_MAPS),
+        fm.type,
+        fm.shortcode,
+    );
     // `buildManifest` records `packageRelative(url, base)`, so the pair it is
     // given has to round-trip. The address is already package-relative, so the
     // honest pair is the address under a base of `"/"` — which strips straight
@@ -141,7 +156,23 @@ export function entriesForNote(fm, name, address, body, ctx) {
     // a consumer resolves the UUID verbatim, and a repository may ship several
     // packs of one type (#1566).
     const uuidFor = (type, id, routeFm) =>
-        id ?
+        // A type this cannot name a single compendium document for has no UUID
+        // to publish, whatever id it derives. That used to follow from such a
+        // note authoring no `id:`; since #270 every addressable note derives
+        // one, so "has an id" stopped being evidence a document exists and the
+        // rule is stated where it belongs — beside the addresses — rather than
+        // resting on an absent field. `collectFoundryEntries` skips such a note
+        // outright; the content index calls this function directly, so the
+        // guard has to live on this side of it.
+        //
+        // Two sets, for opposite reasons (see `note-claims.mjs`). A **homepage**
+        // is in no pack: it compiles to a page and there is nothing to address.
+        // A **folder** may be in several — it materialises in every pack holding
+        // a document that references it (#276) — so no one UUID identifies it,
+        // and its id is hashed under the `folder` namespace against its own
+        // address rather than under `document`. Emitting one would publish an
+        // `Item` UUID for a `Folder`, at an id no document carries.
+        id && !NEVER_PACKED_TYPES.has(String(type)) && !DERIVED_PACKED_TYPES.has(String(type)) ?
             compendiumUuid(
                 foundryPackageId,
                 type,
@@ -152,8 +183,15 @@ export function entriesForNote(fm, name, address, body, ctx) {
             )
         :   undefined;
 
-    if (hasDocEntry(fm.type)) {
-        const docKey = canonicalKey(contentPackage, `doc${fm.type}`, fm.shortcode);
+    const carriesDoc =
+        ctx.docEntryTypes ?
+            ctx.docEntryTypes.has(String(currentType(fm.type)))
+        :   hasDocEntry(fm.type);
+    if (carriesDoc) {
+        // `NO_SYSTEM`, whatever the item is: a documentation journal is a
+        // JournalEntry, which no game system defines, and there is one of them
+        // however many system blocks the note carries.
+        const docKey = canonicalKey(contentPackage, NO_SYSTEM, `doc${fm.type}`, fm.shortcode);
         const docEntryId = fm.id ? itemDocEntryId(fm.id) : undefined;
         const docUuid = uuidFor("doc", docEntryId);
         return [
@@ -214,7 +252,7 @@ export function entriesForNote(fm, name, address, body, ctx) {
  * @returns {{entries: Array<object>, notes: number,
  *   skipped: Array<{file: string, reason: string}>}}
  */
-export function collectManifestEntries(contentBase, ctx) {
+export function collectFoundryEntries(contentBase, ctx) {
     const entries = [];
     const skipped = [];
     // Counted separately because they are genuinely different numbers: an item
@@ -225,6 +263,10 @@ export function collectManifestEntries(contentBase, ctx) {
         skipDirectories: ctx.skipDirectories,
     })) {
         if (!fm) continue;
+        // Its authored pin, or the id derived from its canonical address
+        // (#270). Resolved before anything reads `fm.id`, so the UUID this
+        // pass publishes is the one the pack passes compiled under.
+        resolveNoteId(fm, { pkg: ctx.contentPackage });
         const rel = path.relative(contentBase, absPath);
         assertNoDeclaredPackage(fm, {
             file: rel,
@@ -234,6 +276,7 @@ export function collectManifestEntries(contentBase, ctx) {
         assertNoDraftField(fm, { file: rel, absPath });
         assertNoAliasesField(fm, { file: rel, absPath });
         assertNoSectionField(fm, { file: rel, absPath });
+        assertNoTraitsField(fm, { file: rel, absPath });
         if (!fm.type || !fm.shortcode) continue;
         // A homepage is addressed like every other note since #182, and a
         // shortcode alone would now put it here. It stays out for the reason it
@@ -270,83 +313,37 @@ export function collectManifestEntries(contentBase, ctx) {
  * @returns {{contentPackage: string, foundryPackageId: string, packRouter: object,
  *   web: boolean, skipDirectories: readonly string[]}}
  */
-export function manifestContext(config = loadPackConfig()) {
+export function foundryIdentities(config = loadPackConfig()) {
     return {
         contentPackage: config.contentPackage,
         foundryPackageId: config.foundryPackage,
         packRouter: routerFor(config),
-        web: publishesContentPages(config),
-        // The walk's own configuration, threaded through rather than left to
-        // its default, so a caller that passes a config drives every read.
-        skipDirectories: config.skipDirectories,
+        // Carried in the context rather than read from the global config at the
+        // call site, so the pass really is a pure function of what it is handed
+        // — which is what lets the content index drive the same derivation with
+        // a configuration it resolved itself (#239).
+        docEntryTypes: config.docEntryTypes,
     };
 }
 
 /**
- * Emits this package's link manifest.
+ * The identities an emission runs against, from configuration.
  *
- * One package, because a configuration declares exactly one `contentPackage`
- * and nothing in the surface can express a second. {@link writeManifests} keeps
- * its package→entries map — it is the general writer — but there is no setting
- * here to choose with.
+ * {@link foundryIdentities} plus what only a *manifest* emission needs. The
+ * split is what lets the content index derive the same Foundry addresses from
+ * the same code without also depending on whether the package publishes pages,
+ * which is no part of a UUID (#239).
  *
- * @param {object} [options] - Options.
- * @param {string} [options.contentBase] - The content tree; defaults to the
- *   configured `paths.content`.
- * @param {string} [options.outDir] - Where to write; defaults to the configured
- *   `paths.manifestOut`.
- * @param {object} [options.config] - A resolved configuration; loaded when
- *   omitted.
- * @returns {{written: Array<{package: string, file: string, count: number}>,
- *   entries: number, notes: number,
- *   skipped: Array<{file: string, reason: string}>}}
- * @throws {Error} When the repository does not declare that it publishes a
- *   manifest, when the tree is absent, or when it yields no published note — a
- *   manifest claiming this package publishes nothing is worse than none, since
- *   a consumer reads it as authoritative and turns every link into this package
- *   into a reported typo.
+ * @param {object} [config] - A resolved configuration; loaded when omitted.
+ * @returns {{contentPackage: string, foundryPackageId: string, packRouter: object,
+ *   web: boolean, skipDirectories: readonly string[]}}
  */
-export function emitLinkManifest({ contentBase, outDir, config } = {}) {
-    const resolved = config ?? loadPackConfig();
-    const tree = contentBase ?? resolved.paths.content;
-    const dir = outDir ?? resolved.paths.manifestOut;
-    const ctx = manifestContext(resolved);
-
-    // A repository that has not declared it publishes a manifest must not
-    // produce one: the file is vendored by consumers and read as authoritative,
-    // so emitting it is a statement about this package rather than a local
-    // convenience. Checked here rather than in the command, so a library caller
-    // cannot route around the declaration.
-    if (!resolved.publish.manifests.publish) {
-        throw new Error(
-            `this repository does not publish a link manifest — set ` +
-                `\`publish.manifests.publish: true\` in its content-build ` +
-                `configuration to change that`,
-        );
-    }
-
-    if (!fs.existsSync(tree)) {
-        throw new Error(`no content tree at ${tree}`);
-    }
-
-    const { entries, notes, skipped } = collectManifestEntries(tree, ctx);
-    if (entries.length === 0) {
-        throw new Error(
-            `${tree} yielded no published notes, so the manifest would ` +
-                `claim this package publishes nothing`,
-        );
-    }
-
-    const written = writeManifests(
-        new Map([[ctx.contentPackage, entries]]),
-        dir,
-        // The one surviving role of a base: `undefined` is the statement "this
-        // build publishes no pages", and no entry then carries a `path`
-        // (#1516). The value itself cancels — every address above is already
-        // package-relative — so it is a sentinel, not a location.
-        ctx.web ? { [ctx.contentPackage]: "/" } : undefined,
-        { [ctx.contentPackage]: ctx.foundryPackageId },
-    );
-
-    return { written, entries: entries.length, notes, skipped };
+export function entryContext(config = loadPackConfig()) {
+    return {
+        ...foundryIdentities(config),
+        web: publishesContentPages(config),
+        // The walk's own configuration, threaded through rather than left to
+        // its default, so a caller that passes a config drives every read.
+        skipDirectories: config.skipDirectories,
+    };
 }