@heroiclands/package-build 23.8.0 → 23.9.0

package/engine/language-data.mjs

@@ -160,6 +160,59 @@ export function checkLanguageData(note, { index, types } = {}) {
 
     const findings = [];
 
+    // Only the shape of a supplied declaration: no word is compared with it.
+    const grammar = isMap(data.grammar) ? data.grammar : {};
+    listOf(grammar.plural).forEach((row, i) => {
+        if (row == null || (Array.isArray(row) && row.length === 0))
+            findings.push(
+                finding(
+                    ["data", "grammar", "plural", i],
+                    "error",
+                    `\`data.grammar.plural[${i}]\` must be a map with ending and becomes strings`,
+                ),
+            );
+    });
+    const verbs = grammar.verbs;
+    if (Array.isArray(verbs) && verbs.length === 0)
+        findings.push(
+            finding(
+                ["data", "grammar", "verbs"],
+                "error",
+                "`data.grammar.verbs` must state `invariant: true` or the six `persons` strings",
+            ),
+        );
+    if (isMap(verbs)) {
+        const invariant = Object.hasOwn(verbs, "invariant");
+        const persons = Object.hasOwn(verbs, "persons");
+        if (invariant === persons)
+            findings.push(
+                finding(
+                    ["data", "grammar", "verbs"],
+                    "error",
+                    "`data.grammar.verbs` must state exactly one of `invariant: true` or `persons`",
+                ),
+            );
+        else if (invariant && (verbs.invariant == null || verbs.invariant === false))
+            findings.push(
+                finding(
+                    ["data", "grammar", "verbs", "invariant"],
+                    "error",
+                    "`data.grammar.verbs.invariant` must be true",
+                ),
+            );
+        else if (
+            persons &&
+            (verbs.persons == null || (Array.isArray(verbs.persons) && verbs.persons.length === 0))
+        )
+            findings.push(
+                finding(
+                    ["data", "grammar", "verbs", "persons"],
+                    "error",
+                    "`data.grammar.verbs.persons` must be a map of all six ending strings",
+                ),
+            );
+    }
+
     /**
      * The note an Address names, or a finding where it names none.
      *

package/engine/language-terms.mjs

@@ -43,6 +43,7 @@ export const LANGUAGE_DATA_KEYS = Object.freeze([
     "older",
     "negation",
     "naming",
+    "grammar",
 ]);
 
 /**
@@ -1594,6 +1595,81 @@ function listOfMaps(name, fields, describe) {
     });
 }
 
+/**
+ * The six present persons of the one optional regular verb pattern, in
+ * presentation order. Empty endings leave the citation form unchanged.
+ *
+ * @type {readonly import("./data-keys.mjs").InnerKeySpec[]}
+ */
+export const VERB_PERSON_FIELDS = Object.freeze(
+    [
+        ["I", "I"],
+        ["you", "you"],
+        ["heShe", "he/she"],
+        ["we", "we"],
+        ["youAll", "you all"],
+        ["they", "they"],
+    ].map(([name, label]) =>
+        Object.freeze({
+            name,
+            label,
+            kind: "string",
+            required: true,
+            allowEmpty: true,
+            shape: "an ending string, empty for unchanged",
+            describe: `The ending added to the citation form for ${label}.`,
+        }),
+    ),
+);
+
+/** The two alternatives of a supplied optional verb pattern. */
+const VERB_PATTERN_FIELDS = Object.freeze([
+    Object.freeze({
+        name: "invariant",
+        kind: "boolean",
+        shape: "true, as the alternative to persons",
+        describe: "True where verbs keep the citation form for every person; excludes `persons`.",
+    }),
+    Object.freeze({
+        name: "persons",
+        kind: "map",
+        shape: "a map — `{ I, you, heShe, we, youAll, they }` of ending strings",
+        fields: VERB_PERSON_FIELDS,
+        describe: "All six present endings, each added to the citation form; excludes `invariant`.",
+    }),
+]);
+
+/** The optional, display-only noun and verb patterns of one language. */
+const GRAMMAR_FIELDS = Object.freeze([
+    innerList(
+        "plural",
+        Object.freeze(
+            ["ending", "becomes"].map((name) =>
+                Object.freeze({
+                    name,
+                    kind: "string",
+                    required: true,
+                    allowEmpty: true,
+                    shape: "a literal ending string, which may be empty",
+                    describe:
+                        name === "ending" ?
+                            "The bare suffix replaced by this rule; empty matches every noun."
+                        :   "The bare replacement; empty removes the matching suffix.",
+                }),
+            ),
+        ),
+        "Default noun plurals, replacing the first matching literal ending in authored order. " +
+            "A word's explicit plural wins; no word is checked against this rule.",
+    ),
+    Object.freeze({
+        name: "verbs",
+        kind: "map",
+        shape: "a map — `{ invariant: true }` or `{ persons: { I, you, heShe, we, youAll, they } }`",
+        fields: VERB_PATTERN_FIELDS,
+        describe: "One optional regular present pattern, printed once at the dictionary head.",
+    }),
+]);
+
 /**
  * The `data` fields this module declares, in the order the vocabulary lists
  * them. {@link module:engine/language-data.checkLanguageData} checks what the
@@ -1602,6 +1678,16 @@ function listOfMaps(name, fields, describe) {
  * @type {readonly object[]}
  */
 export const LANGUAGE_DATA_FIELDS = Object.freeze([
+    Object.freeze({
+        name: "grammar",
+        kind: "map",
+        shape: "a map — `{ plural?, verbs? }`",
+        fields: GRAMMAR_FIELDS,
+        describe:
+            "Optional noun plurals and one verb pattern for dictionary presentation. Only the " +
+            "shape of supplied keys is checked; no word must follow or state a rule. A language " +
+            "skill note only.",
+    }),
     listOfMaps(
         "tongues",
         TONGUE_FIELDS,

package/engine/language-words.mjs

@@ -36,7 +36,7 @@
  *   an ancestor's;
  * - every source language resolves, and names a language skill;
  * - a pronunciation is a respelling;
- * - a `built` piece no word or affix of the language glosses is a warning.
+ * - a `built` piece no word, affix or element of the language glosses is a warning.
  *
  * {@link checkLanguageFamily} owns `data.family`, the parent language a
  * daughter reads words from: it is stated only on a language, names a
@@ -285,7 +285,7 @@ export function checkLanguageWords(note, { index } = {}) {
                 finding(
                     [...at("built"), j],
                     "warning",
-                    `the piece \`${piece}\` of \`${word.form}\` matches no word or affix of this ` +
+                    `the piece \`${piece}\` of \`${word.form}\` matches no word, affix or element of this ` +
                         "language or its ancestors, so it prints with no gloss",
                 ),
             );

package/engine/lexical-sections.mjs

@@ -0,0 +1,247 @@
+/*
+ * This file is part of the Song of Heroic Lands (SoHL) system for Foundry VTT.
+ * Copyright (c) 2024-2026 Tom Rodriguez ("Toasty") — <toasty@heroiclands.org>
+ *
+ * This work is licensed under the GNU General Public License v3.0 (GPLv3).
+ * You may copy, modify, and distribute it under the terms of that license.
+ *
+ * For full terms, see the LICENSE.md file in the project root or visit:
+ * https://www.gnu.org/licenses/gpl-3.0.html
+ *
+ * SPDX-License-Identifier: GPL-3.0-or-later
+ */
+
+/**
+ * Automatic dictionaries and glossaries, derived synchronously from the same
+ * rows their SQL layouts read. An existing anchored section receives its list
+ * after its authored text; otherwise the note receives a new final section.
+ * The insertion and the publisher read the same Markdown, on every surface.
+ *
+ * @module
+ */
+
+import { renderAddress } from "./address.mjs";
+import { encodeAddresses } from "./address-values.mjs";
+import { scanBlocks } from "./content-blocks.mjs";
+import { renderDictionary, renderGlossary } from "./dictionary-layout.mjs";
+import { DICTIONARY_COLUMNS, TERM_COLUMNS, dictionaryAndTermRows } from "./dictionary-rows.mjs";
+import { glossaryLanguage } from "./glossary-language.mjs";
+import { scanHeadingAttributes } from "./heading-attributes.mjs";
+import { isNoteRecord } from "./index-records.mjs";
+
+/** The automatic lexical sections and the authored anchors each may reuse. */
+export const LEXICAL_SECTIONS = Object.freeze(
+    [
+        {
+            slug: "dictionary",
+            heading: "Dictionary",
+            aliases: ["dictionary", "lexicon"],
+            dataKey: "words",
+            types: [{ type: "skill", subType: "language" }],
+        },
+        {
+            slug: "glossary",
+            heading: "Glossary",
+            aliases: ["glossary"],
+            dataKey: "glossary",
+            types: [{ type: "lore", subType: "culture" }, { type: "affiliation" }],
+        },
+    ].map((section) =>
+        Object.freeze({
+            ...section,
+            aliases: Object.freeze(section.aliases),
+            types: Object.freeze(section.types.map((kind) => Object.freeze(kind))),
+        }),
+    ),
+);
+
+/** The lexical section this note's own nonempty frontmatter requests. */
+function sectionFor(record) {
+    return LEXICAL_SECTIONS.find(
+        (section) =>
+            section.types.some(
+                (kind) =>
+                    record?.type === kind.type &&
+                    (!kind.subType || record.subType === kind.subType),
+            ) &&
+            Array.isArray(record.data?.[section.dataKey]) &&
+            record.data[section.dataKey].length > 0,
+    );
+}
+
+/**
+ * Whether frontmatter supplies an automatic lexical section, including on a
+ * note with no authored prose. Such a note has content and is not a stub.
+ *
+ * @param {object|null|undefined} frontmatter - A note or its index record.
+ * @returns {boolean} Whether its own nonempty words or glossary qualify.
+ */
+export function hasLexicalEntries(frontmatter) {
+    return Boolean(sectionFor(frontmatter));
+}
+
+/** A note's canonical identity, including a data-only note that was a stub. */
+const keyOf = (record) =>
+    record?.package && record.type && record.shortcode ?
+        renderAddress({
+            package: record.package,
+            system: "note",
+            type: record.type,
+            shortcode: record.shortcode,
+        })
+    :   undefined;
+
+/**
+ * One automatic section, whose rows are rendered at its containing heading's
+ * level when inserted.
+ *
+ * @typedef {object} LexicalSection
+ * @property {string} slug - The canonical section anchor and layout name.
+ * @property {string} heading - The heading used when none is authored.
+ * @property {readonly string[]} aliases - Authored heading anchors to reuse.
+ * @property {{columns: string[], rows: object[]}} result - The selected rows.
+ * @property {import("./dictionary-layout.mjs").LayoutOptions} options - Labels,
+ *   linkability and the page's language, shared by every surface.
+ */
+
+/**
+ * Automatic lexical sections by owning note's canonical Address. Only
+ * nonempty own `words` or `glossary` arrays qualify. A glossary selects only
+ * explicitly declared owner entries, including a rank only when named there.
+ *
+ * @param {object[]} records - The audience's visible index records.
+ * @param {object[]} [all=records] - Complete records for lineage traversal.
+ * @returns {Map<string, LexicalSection[]>} Sections by owning Address.
+ */
+export function lexicalSectionsFor(records, all = records) {
+    const visible = encodeAddresses(records).filter(isNoteRecord);
+    const complete = encodeAddresses(all).filter(isNoteRecord);
+    const byKey = new Map(visible.map((record) => [keyOf(record), record]));
+    const addressContext = {
+        types: new Set(complete.map((record) => record.type)),
+        packages: new Set(complete.map((record) => record.package)),
+        system: "note",
+    };
+    const { dictionary, terms } = dictionaryAndTermRows(visible, complete);
+    const sections = new Map();
+    for (const [key, record] of byKey) {
+        if (!key) continue;
+        const spec = sectionFor(record);
+        if (!spec) continue;
+        const language = spec.slug === "dictionary";
+        const rows =
+            language ?
+                dictionary.filter((row) => row.language === key)
+            :   terms.filter((row) => row.owner === key && row.origins.includes("glossary"));
+        const pageLanguage = glossaryLanguage(undefined, record, {
+            self: key,
+            keyOf: (value) => (typeof value === "string" ? value : undefined),
+            lookup: (address) => byKey.get(address),
+        });
+        sections.set(key, [
+            {
+                slug: spec.slug,
+                heading: spec.heading,
+                aliases: spec.aliases,
+                result: { columns: [...(language ? DICTIONARY_COLUMNS : TERM_COLUMNS)], rows },
+                options: {
+                    addressContext: { ...addressContext, package: record.package },
+                    pageLanguage,
+                    pageOwner: key,
+                    ...(language ? { verbs: record.data?.grammar?.verbs } : {}),
+                    labelOf: (address) => byKey.get(address)?.name?.full,
+                    linkable: (ref) => {
+                        const target = byKey.get(renderAddress({ ...ref, system: "note" }));
+                        return Boolean(target && (target.address || hasLexicalEntries(target)));
+                    },
+                },
+            },
+        ]);
+    }
+    return sections;
+}
+
+/** Authored section headings outside literal content and named containers. */
+function headings(markdown) {
+    const containers = scanBlocks(markdown).blocks;
+    return scanHeadingAttributes(markdown)
+        .headings.filter(
+            (heading) =>
+                !containers.some(
+                    (block) => heading.line - 1 >= block.start && heading.line - 1 <= block.end,
+                ),
+        )
+        .map((heading) => ({ line: heading.line - 1, level: heading.level, slug: heading.id }));
+}
+
+/**
+ * Insert lists at the end of their authored anchored sections, or append a
+ * new H1 section when no heading names one. Source lines after an insertion
+ * retain their original positions; inserted lines point to their heading.
+ *
+ * @param {string} markdown - The body with authored directives expanded.
+ * @param {LexicalSection[]} [sections=[]] - From {@link lexicalSectionsFor}.
+ * @param {{authored?: string, lineMap?: object[], generatedFrom?: number}} [context] -
+ *   The original authored body, source mappings and the start of appended
+ *   nonlexical generated sections. Without `authored`, `markdown` is the source.
+ * @returns {{markdown: string, lineMap: object[], generatedFrom?: number}}
+ */
+export function insertLexicalSections(
+    markdown,
+    sections = [],
+    { authored = markdown, lineMap, generatedFrom } = {},
+) {
+    const lines = String(markdown ?? "").split("\n");
+    const locations = lineMap ? [...lineMap] : lines.map((_, line) => ({ line, generated: false }));
+    if (!sections.length)
+        return {
+            markdown: lines.join("\n"),
+            lineMap: locations,
+            ...(generatedFrom === undefined ? {} : { generatedFrom }),
+        };
+    const edits = [];
+    const realHeadings = headings(String(authored ?? ""))
+        .map((heading) => ({
+            ...heading,
+            sourceLine: heading.line,
+            line: locations.findIndex(
+                (location, at) =>
+                    location.line === heading.line &&
+                    !location.generated &&
+                    (generatedFrom === undefined || at < generatedFrom),
+            ),
+        }))
+        .filter((heading) => heading.line >= 0);
+    for (const section of sections) {
+        const own = realHeadings.find((heading) => section.aliases.includes(heading.slug));
+        const next =
+            own &&
+            realHeadings.find((heading) => heading.line > own.line && heading.level <= own.level);
+        const at = next?.line ?? (own ? (generatedFrom ?? lines.length) : lines.length);
+        const level = own?.level ?? 1;
+        const render = section.slug === "dictionary" ? renderDictionary : renderGlossary;
+        const list = render(section.result, {
+            ...section.options,
+            sectionLevel: Math.min(6, level + 1),
+        });
+        if (!list && own) continue;
+        const text = `${own ? "" : `# ${section.heading} {#${section.slug}}\n\n`}${list}`;
+        edits.push({
+            at,
+            source: own?.sourceLine ?? Math.max(0, String(authored ?? "").split("\n").length - 1),
+            lines: ["", ...text.split("\n"), ""],
+        });
+    }
+    for (const edit of edits.sort((a, b) => b.at - a.at)) {
+        const location = { line: edit.source, generated: true };
+        lines.splice(edit.at, 0, ...edit.lines);
+        locations.splice(edit.at, 0, ...edit.lines.map(() => ({ ...location })));
+        if (generatedFrom !== undefined && edit.at <= generatedFrom)
+            generatedFrom += edit.lines.length;
+    }
+    return {
+        markdown: lines.join("\n"),
+        lineMap: locations,
+        ...(generatedFrom === undefined ? {} : { generatedFrom }),
+    };
+}

package/engine/note-claims.mjs

@@ -105,6 +105,7 @@ import { HOMEPAGE_TYPE } from "./homepage.mjs";
 import { FOLDER_TYPE } from "./folder-notes.mjs";
 import { BUNDLE_TYPE } from "./bundle-notes.mjs";
 import { NOTE_VOCABULARY } from "./note-vocabulary.mjs";
+import { hasLexicalEntries } from "./lexical-sections.mjs";
 
 /**
  * Note types that compile into **no compendium document, by design**.
@@ -769,6 +770,7 @@ export function unclaimedNoteFindings(
         // claiming this note", because the answer there is yes.
         const produces = documentClassesFor(type, resolved, {
             hasProse: () => {
+                if (hasLexicalEntries(fm)) return true;
                 const source = parsed?.get(absPath);
                 return source ? Boolean(source.body) : noteHasProse(absPath);
             },

package/engine/note-state.mjs

@@ -14,7 +14,7 @@
 /**
  * The three states of a note, and the one observable fact they rest on.
  *
- * A note whose body is empty is a **stub**: frontmatter only — identity and
+ * A note whose body is empty and has no automatic lexical content is a **stub**: frontmatter only — identity and
  * statistics — reaching the content index and queryable there, generating no
  * page, no address and no link target. A note with a body and the declared
  * `draft` tag is a **draft**: it publishes, compiles and resolves like any
@@ -39,6 +39,7 @@
  */
 
 import { NOTE_VOCABULARY } from "./note-vocabulary.mjs";
+import { hasLexicalEntries } from "./lexical-sections.mjs";
 
 /**
  * The three states, in the order they read as a ladder.
@@ -97,7 +98,8 @@ export function isStubbableType(type, vocabulary = NOTE_VOCABULARY) {
 }
 
 /**
- * Whether a note is a stub: an empty body on a type an empty body suppresses.
+ * Whether a note is a stub: an empty body on a type an empty body suppresses,
+ * without an automatic dictionary or glossary supplied by its frontmatter.
  *
  * The one place the two halves are put together, so the index, the lint and the
  * guard cannot answer it differently.
@@ -109,7 +111,11 @@ export function isStubbableType(type, vocabulary = NOTE_VOCABULARY) {
  * @returns {boolean} Whether the note is a stub.
  */
 export function isStubNote(frontmatter, body, vocabulary = NOTE_VOCABULARY) {
-    return isEmptyBody(body) && isStubbableType(frontmatter?.type, vocabulary);
+    return (
+        isEmptyBody(body) &&
+        !hasLexicalEntries(frontmatter) &&
+        isStubbableType(frontmatter?.type, vocabulary)
+    );
 }
 
 /**

package/engine/note-vocabulary.mjs

@@ -600,7 +600,7 @@ const WORD_DATA_FIELDS = Object.freeze([
             fields: WORD_FIELDS,
         }),
         describe:
-            "The language's words, one map each, in the order they print. A language skill " +
+            "The words this language states, one map each, in authored order; its automatic dictionary sorts them alphabetically. A language skill " +
             "note only. See [a language's words](format-details.md#a-languages-words).",
     }),
     Object.freeze({
@@ -671,9 +671,10 @@ const GLOSSARY_FIELD = Object.freeze({
         fields: GLOSSARY_ENTRY_FIELDS,
     }),
     describe:
-        "Its own glossary terms: euphemisms, idioms, terms of art and titles that are not " +
-        "ranks. An entry with a `definition` creates a term or replaces its senses; one " +
-        "without marks a term the note already has. Every entry resolves to a language. " +
+        "Its own glossary entries: common vocabulary, dictionary words with a contextual " +
+        "meaning, and phrases, compounds or terms of art. A named dictionary word supplies " +
+        "its senses and class unless `definition` supplies the sense. Common marks belong " +
+        "to this owner. Every entry resolves to a language. " +
         "Empty when unstated. See [glossary terms](format-details.md#glossary-terms).",
 });
 
@@ -1820,8 +1821,8 @@ export const NOTE_VOCABULARY = Object.freeze({
                 ...LANGUAGE_FIELD,
                 describe:
                     "The trade, temple or order tongue the affiliation keeps, as a language " +
-                    "skill note. It reaches the affiliation's culture, and its words enter that " +
-                    "culture's glossary. A glossary entry that states no `language` is in it.",
+                    "skill note. A glossary entry that states no `language` is in it. The " +
+                    "affiliation chooses its own common vocabulary and context meanings.",
             },
             GLOSSARY_FIELD,
             {

package/engine/pdf-build.mjs

@@ -1013,7 +1013,15 @@ async function prepareWorld(resolved, { contentBase, findings, resolveReplacemen
         scheme,
     };
     const collected = collectContentPages(contentBase, ctx);
-    const gates = siteGates(collected.pages, collected, { config: resolved, records });
+    const sqlTables = await prepareTreeSqlTables(contentBase, {
+        config: resolved,
+        skipDirectories: resolved.skipDirectories,
+        records,
+        parsed,
+        database: db,
+        audience: "public",
+    });
+    const gates = siteGates(collected.pages, collected, { config: resolved, records, sqlTables });
     if (gatesFailed(gates)) {
         for (const [name, list] of Object.entries(gates)) {
             if (!Array.isArray(list)) continue;
@@ -1030,14 +1038,6 @@ async function prepareWorld(resolved, { contentBase, findings, resolveReplacemen
 
     const byFile = new Map(collected.pages.map((page) => [page.file, page]));
     const universe = tableUniverse(collected.pages);
-    const sqlTables = await prepareTreeSqlTables(contentBase, {
-        config: resolved,
-        skipDirectories: resolved.skipDirectories,
-        records,
-        parsed,
-        database: db,
-        audience: "public",
-    });
     // The address space an `![[…]]` embed resolves against — the same one the
     // site builds, so one authored picture reaches both surfaces or neither.
     const assets = assetAddressIndex(records, {

package/engine/site-build.mjs

@@ -430,9 +430,11 @@ export function writeHomepages(outRoot, pages, config, { related, records, forei
  * @param {object} options
  * @param {object} options.config - The resolved build configuration.
  * @param {object[]} options.records - Content-index records.
+ * @param {Map<string, object[]>} [options.sqlTables] - Prepared directives,
+ *   whose emitted anchors belong to the pages they expand in.
  * @returns {object} The gate results and, when they pass, the built index.
  */
-export function siteGates(pages, findings, { config, records }) {
+export function siteGates(pages, findings, { config, records, sqlTables }) {
     const out = {
         // Always empty here: the homepage count is decided in `buildSite`
         // before the content walk, and a failing count returns without ever
@@ -464,7 +466,18 @@ export function siteGates(pages, findings, { config, records }) {
     out.unaddressable = unaddressableForeignPackages(foreign.index);
     if (out.unaddressable.length) return out;
 
-    out.index = buildSiteIndex(pages, {
+    const expanded =
+        sqlTables ?
+            pages.map((page) => ({
+                ...page,
+                body: expandContentTables(page.body ?? "", {
+                    source: page.file,
+                    sqlTables: sqlTables.get(page.file),
+                    pageLists: sqlTables.pageLists?.get(page.file),
+                }).markdown,
+            }))
+        :   pages;
+    out.index = buildSiteIndex(expanded, {
         package: config.contentPackage,
         foreignIndex: foreign.index,
         foreignReferences: foreign.references,
@@ -1367,6 +1380,7 @@ export function buildSite({ config, sqlTables } = {}) {
     const gates = siteGates([...pages, ...homepageEntries], content, {
         config: resolved,
         records: ctx.records,
+        sqlTables,
     });
     if (gatesFailed(gates)) {
         return {