@heroiclands/package-build 23.6.1 → 23.7.1

package/engine/language-words.mjs

@@ -26,8 +26,15 @@
  * - a word's `field` is one of `data.fields`;
  * - a `borrowed` source names its `language`;
  * - no form stands twice in one language;
- * - every `attested` Address and source language resolves, and a source
- *   language names a language skill;
+ * - a word's `gloss` states at least one sense;
+ * - an affix that `replaces` the stem's close does not also `lengthen` it;
+ * - an affix states `given` only where its `kind` is `name`;
+ * - an affix states a `form` or `zero: true`, never both;
+ * - an element's `word` and the particle of `data.negation` are forms of the
+ *   words of the language or an ancestor;
+ * - a word's `frame` names a frame of `data.sounds.frames`, its language's or
+ *   an ancestor's;
+ * - every source language resolves, and names a language skill;
  * - a pronunciation is a respelling;
  * - a `built` piece no word or affix of the language glosses is a warning.
  *
@@ -48,7 +55,9 @@ import { addressKeyOf, ownKey } from "./culture-reach.mjs";
 import {
     BORROWED_SOURCE_KIND,
     LANGUAGE_SUBTYPE,
+    NAME_AFFIX_KIND,
     WORD_DATA_KEYS,
+    glossSenses,
     pronunciationProblem,
 } from "./word-terms.mjs";
 import { WORD_CLASS_MODIFIERS, parseWordClass } from "./word-classes.mjs";
@@ -174,6 +183,13 @@ export function checkLanguageWords(note, { index } = {}) {
             }).map((key) => dataAt(key))
         :   [];
     const glossOf = pieceGlosses(data, ...ancestors);
+    // The frames a word may be set in: its language's, then each ancestor's.
+    const frames = new Set();
+    for (const one of [data, ...ancestors]) {
+        const sounds = isMap(one?.sounds) ? one.sounds : {};
+        for (const frame of listOf(sounds.frames))
+            if (isMap(frame) && typeof frame.frame === "string") frames.add(frame.frame);
+    }
     const used = new Set();
     const forms = new Map();
     listOf(data.words).forEach((word, i) => {
@@ -236,10 +252,20 @@ export function checkLanguageWords(note, { index } = {}) {
             findings.push(...languageFindings(word.source.language, [...at("source"), "language"]));
         }
 
-        listOf(word.attested).forEach((value, j) => {
-            for (const result of addressFindings(value, [...at("attested"), j], undefined))
-                if (!result.target) findings.push(result);
-        });
+        if (typeof word.frame === "string" && !frames.has(word.frame))
+            findings.push(
+                finding(
+                    at("frame"),
+                    "error",
+                    `the frame \`${word.frame}\` of \`${word.form}\` is not one of this language's ` +
+                        "`data.sounds.frames`",
+                ),
+            );
+
+        if (Array.isArray(word.gloss) && !glossSenses(word.gloss).length)
+            findings.push(
+                finding(at("gloss"), "error", "a `gloss` list states at least one sense"),
+            );
 
         if (typeof word.pronunciation === "string") {
             const problem = pronunciationProblem(word.pronunciation);
@@ -266,6 +292,89 @@ export function checkLanguageWords(note, { index } = {}) {
         });
     });
 
+    listOf(data.affixes).forEach((affix, i) => {
+        if (!isMap(affix)) return;
+        const form = typeof affix.form === "string" && affix.form.trim() !== "";
+        const zero = affix.zero === true;
+        if (form && zero)
+            findings.push(
+                finding(
+                    ["data", "affixes", i, "zero"],
+                    "error",
+                    `\`data.affixes[${i}]\` states a \`form\` and \`zero: true\`; a zero affix states no form`,
+                    true,
+                ),
+            );
+        else if (!form && !zero && affix.form == null)
+            findings.push(
+                finding(
+                    ["data", "affixes", i],
+                    "error",
+                    `\`data.affixes[${i}]\` states neither a \`form\` nor \`zero: true\``,
+                ),
+            );
+    });
+
+    listOf(data.affixes).forEach((affix, i) => {
+        if (!isMap(affix) || affix.given == null || affix.kind === NAME_AFFIX_KIND) return;
+        findings.push(
+            finding(
+                ["data", "affixes", i, "given"],
+                "error",
+                "`given` says who a given name belongs to, and " +
+                    (typeof affix.kind === "string" ?
+                        `\`data.affixes[${i}]\` is of kind \`${affix.kind}\``
+                    :   `\`data.affixes[${i}]\` is of no kind`) +
+                    `; it is stated on an affix of kind \`${NAME_AFFIX_KIND}\` only`,
+                true,
+            ),
+        );
+    });
+
+    // The forms an element's `word` and the negative particle may name.
+    const wordForms = new Set();
+    for (const one of [data, ...ancestors])
+        for (const word of listOf(one?.words))
+            if (isMap(word) && typeof word.form === "string") wordForms.add(word.form);
+    listOf(data.elements).forEach((element, i) => {
+        if (!isMap(element) || typeof element.word !== "string" || wordForms.has(element.word))
+            return;
+        findings.push(
+            finding(
+                ["data", "elements", i, "word"],
+                "error",
+                `the word \`${element.word}\` of the element \`${element.form}\` is not a form of ` +
+                    "this language's words or its ancestors'",
+            ),
+        );
+    });
+    if (
+        isMap(data.negation) &&
+        typeof data.negation.particle === "string" &&
+        !wordForms.has(data.negation.particle)
+    )
+        findings.push(
+            finding(
+                ["data", "negation", "particle"],
+                "error",
+                `the negative particle \`${data.negation.particle}\` is not a form of this ` +
+                    "language's words or its ancestors'",
+            ),
+        );
+
+    listOf(data.affixes).forEach((affix, i) => {
+        if (!isMap(affix) || affix.lengthens !== true || affix.replaces == null) return;
+        findings.push(
+            finding(
+                ["data", "affixes", i, "lengthens"],
+                "error",
+                `\`${affix.form}\`: an affix that replaces the stem's close does not also lengthen ` +
+                    "its last vowel; state one",
+                true,
+            ),
+        );
+    });
+
     for (const [token, i] of declared)
         if (!used.has(token))
             findings.push(

package/engine/note-addresses.mjs

@@ -69,15 +69,49 @@ export const STRUCTURED_ADDRESSES = Object.freeze([
         type: "place",
         accepts: ["place"],
     },
-    // A language's word Addresses and a glossary entry's language, read
-    // leniently: a value that is no Address stays as written, for the words or
+    // A language's Addresses and a glossary entry's language, read leniently:
+    // a value that is no Address stays as written, for the language or
     // glossary check to refuse at its own key.
     {
         noteType: "skill",
-        path: ["data", "words", "*", "attested"],
-        shape: "list",
+        path: ["data", "tongues", "*", "language"],
+        shape: "value",
+        type: "skill",
+        accepts: ["skill"],
+        lenient: true,
+    },
+    ...["from", "to"].map((key) => ({
+        noteType: "skill",
+        path: ["data", "soundChanges", "*", key],
+        shape: "value",
+        type: "skill",
+        accepts: ["skill"],
+        lenient: true,
+    })),
+    {
+        noteType: "skill",
+        path: ["data", "cognates", "*", "forms"],
+        shape: "keys",
+        type: "skill",
+        accepts: ["skill"],
         lenient: true,
     },
+    ...[
+        ["folders", "folder", ["folder"], "none"],
+        ["cultures", "lore", ["lore"]],
+        ["lore", "lore", ["lore"]],
+        ["parents", "place", undefined],
+        ["addresses", undefined, undefined],
+        ["skip", undefined, undefined],
+    ].map(([kind, type, accepts, system]) => ({
+        noteType: "skill",
+        path: ["data", "scope", kind],
+        shape: "list",
+        ...(type ? { type } : {}),
+        ...(accepts ? { accepts } : {}),
+        ...(system ? { system } : {}),
+        lenient: true,
+    })),
     {
         noteType: "skill",
         path: ["data", "words", "*", "source", "language"],

package/engine/note-vocabulary.mjs

@@ -132,6 +132,8 @@ import {
     checkLanguageWords,
 } from "./language-words.mjs";
 import { checkGlossary } from "./glossary-entries.mjs";
+import { LANGUAGE_DATA_FIELDS } from "./language-terms.mjs";
+import { checkLanguageData } from "./language-data.mjs";
 import { checkAffiliationReach } from "./culture-reach.mjs";
 import { checkLanguageNames, requiredGivenGenders } from "./language-names.mjs";
 import { SOCIAL_TIES, SOCIAL_TIE_TARGET_TYPES } from "./social-tie-terms.mjs";
@@ -1989,8 +1991,12 @@ export const NOTE_VOCABULARY = Object.freeze({
     skill: Object.freeze({
         stubbable: true,
         // A language note's words, held to the master word classes and to what
-        // the note declares; no other skill states them.
-        check: checkLanguageWords,
+        // the note declares, and its register, sounds and cognates held to
+        // what they name; no other skill states them.
+        check: (note, context) => [
+            ...checkLanguageWords(note, context),
+            ...checkLanguageData(note, { ...context, types: Object.keys(NOTE_VOCABULARY) }),
+        ],
         subTypes: Object.freeze([
             "social",
             "nature",
@@ -2017,6 +2023,7 @@ export const NOTE_VOCABULARY = Object.freeze({
             FAMILY_FIELD,
             ABBREVIATION_FIELD,
             ...WORD_DATA_FIELDS,
+            ...LANGUAGE_DATA_FIELDS,
         ]),
     }),
 

package/engine/sql-tables.mjs

@@ -44,6 +44,7 @@ import {
     wordRows,
 } from "./word-rows.mjs";
 import { DICTIONARY_COLUMNS, TERM_COLUMNS, dictionaryAndTermRows } from "./dictionary-rows.mjs";
+import { LANGUAGE_COLUMN_TYPES, LANGUAGE_RELATIONS } from "./language-rows.mjs";
 import { renderDictionary, renderGlossary } from "./dictionary-layout.mjs";
 import { glossaryLanguage } from "./glossary-language.mjs";
 import { eventViewSections } from "./event-views.mjs";
@@ -99,12 +100,13 @@ export const SQL_FENCE_ATTRIBUTES = Object.freeze({
             "letter of a dictionary or a glossary.",
     }),
     key: Object.freeze({
-        value: "`true` or `false`",
+        value: "`true`, `false` or `only`",
         default: "`true`",
         summary:
             "Whether a dictionary or a glossary prints its key, the line of abbreviations it " +
             "uses. `false` leaves the key out, for a page that states it once above several " +
-            "fences.",
+            "fences; `only` prints the key for the fence's rows and nothing else, which is how " +
+            "that page states it. Refused on a fence set as a table.",
     }),
     layout: Object.freeze({
         value: "`table`, `dictionary` or `glossary`",
@@ -135,7 +137,7 @@ const escapeCell = (text) =>
  *
  * @param {string} markdown - The note body, frontmatter already stripped.
  * @returns {Array<{line: number, close: number, indent: string, query: string,
- *   allowEmpty: boolean, sectionLevel: number, showKey: boolean, layout: string, block: string}>} One entry per
+ *   allowEmpty: boolean, sectionLevel: number, showKey: boolean|"only", layout: string, block: string}>} One entry per
  *   fence, in document order.
  */
 export function findSqlBlocks(markdown) {
@@ -170,19 +172,22 @@ export function findSqlBlocks(markdown) {
                 errors.push(error.message);
             }
         }
+        /** @type {boolean|"only"} */
         let showKey = true;
         if (Object.hasOwn(args, "key")) {
-            try {
-                showKey = booleanAttribute(args.key, "key");
-            } catch (error) {
-                errors.push(error.message);
-            }
+            if (args.key === "only") showKey = "only";
+            else if (args.key === "true" || args.key === "false") showKey = args.key === "true";
+            else errors.push("key needs the literal value true, false or only");
         }
         if (!Number.isInteger(level) || level < 1 || level > 6)
             errors.push("section-level needs an integer from 1 through 6");
         const layout = String(args.layout ?? "table");
         if (!SQL_LAYOUTS.includes(layout))
             errors.push(`layout is \`table\`, \`dictionary\` or \`glossary\`, not \`${layout}\``);
+        else if (Object.hasOwn(args, "key") && layout === "table")
+            errors.push(
+                "key sets a dictionary's or a glossary's key, and this fence's layout is `table`",
+            );
         blocks.push({
             line: i,
             close,
@@ -337,6 +342,7 @@ export async function openNotesDatabase(
         "language_lineage",
         "dictionary",
         "terms",
+        ...LANGUAGE_RELATIONS.map(([name]) => name),
     ])
         await connection.run(
             `CREATE VIEW ${WITH_STUBS_SCHEMA}.${relation} AS SELECT * FROM main.${relation}`,
@@ -556,7 +562,9 @@ async function createTableFromFile(connection, name, select) {
  * {@link module:engine/word-rows.GLOSSARY_COLUMNS},
  * {@link module:engine/word-rows.LINEAGE_COLUMNS},
  * {@link module:engine/dictionary-rows.DICTIONARY_COLUMNS} and
- * {@link module:engine/dictionary-rows.TERM_COLUMNS} name.
+ * {@link module:engine/dictionary-rows.TERM_COLUMNS} name; then each relation
+ * of a language's register, sounds, history and names, as
+ * {@link module:engine/language-rows} derives them.
  *
  * Derived from the same records `entries` reads, stubs included, and a GM
  * note's rows are left out on a public surface exactly as the note is, while a
@@ -623,6 +631,15 @@ async function createWordRelations(connection, stem, records, prefix, audience)
         `${prefix}terms`,
         TERM_COLUMNS,
     );
+    for (const [name, rowsOf, columns] of LANGUAGE_RELATIONS)
+        await createRowRelation(
+            connection,
+            `${stem}-${name}.jsonl`,
+            rowsOf(visible, records),
+            `${prefix}${name}`,
+            columns,
+            LANGUAGE_COLUMN_TYPES[name],
+        );
 }
 
 /**
@@ -635,19 +652,35 @@ async function createWordRelations(connection, stem, records, prefix, audience)
  * @param {object[]} rows - The rows, each carrying every column.
  * @param {string} view - The qualified view name.
  * @param {readonly string[]} columns - Its columns, for the empty case.
+ * @param {Readonly<Record<string, string>>} [types] - Each column's type, for
+ *   the empty case; a column it leaves out is text. Without it, a whole-number
+ *   column is a number and every other column text. A `DOUBLE` column is cast
+ *   whatever the rows hold, because a fraction written whole reads as an
+ *   integer.
  * @returns {Promise<void>}
  */
-async function createRowRelation(connection, file, rows, view, columns) {
+async function createRowRelation(connection, file, rows, view, columns, types) {
     if (!rows.length) {
         const typed = columns.map((column) => {
-            const type = ROW_INTEGER_COLUMNS.includes(column) ? "BIGINT" : "VARCHAR";
+            const type =
+                types ? (types[column] ?? "VARCHAR")
+                : ROW_INTEGER_COLUMNS.includes(column) ? "BIGINT"
+                : "VARCHAR";
             return `NULL::${type} AS "${column}"`;
         });
         await connection.run(`CREATE VIEW ${view} AS SELECT ${typed.join(", ")} WHERE false`);
         return;
     }
     fs.writeFileSync(file, rows.map((row) => JSON.stringify(row)).join("\n"));
-    await createTableFromFile(connection, view, readJsonAuto(file));
+    const fractions = Object.entries(types ?? {})
+        .filter(([, type]) => type === "DOUBLE")
+        .map(([column]) => `CAST("${column}" AS DOUBLE) AS "${column}"`);
+    const source = readJsonAuto(file);
+    await createTableFromFile(
+        connection,
+        view,
+        fractions.length ? `SELECT * REPLACE (${fractions.join(", ")}) FROM (${source})` : source,
+    );
 }
 
 /**

package/engine/word-citations.mjs

@@ -0,0 +1,206 @@
+/*
+ * This file is part of the Song of Heroic Lands (SoHL) system for Foundry VTT.
+ * Copyright (c) 2024-2026 Tom Rodriguez ("Toasty") — <toasty@heroiclands.org>
+ *
+ * This work is licensed under the GNU General Public License v3.0 (GPLv3).
+ * You may copy, modify, and distribute it under the terms of that license.
+ *
+ * For full terms, see the LICENSE.md file in the project root or visit:
+ * https://www.gnu.org/licenses/gpl-3.0.html
+ *
+ * SPDX-License-Identifier: GPL-3.0-or-later
+ */
+
+/**
+ * The notes that cite a language's words, read from the notes' prose and from
+ * the names their frontmatter carries.
+ *
+ * **A marked use is an italic span of a note's body**: `_vazhmadêr_` or
+ * `*vazhmadêr*`, holding text alone — a span holding a link, a code span or
+ * other markup is no use, and neither is anything in a code fence. A use cites
+ * every word, of every language note of the package, whose form or one of
+ * whose `variants` it spells, compared with diacritics taken off and case
+ * folded, so `_Vazhmadêr_` and `_vazhmader_` both cite `vazhmadêr`. An
+ * italic span that spells no word cites nothing.
+ *
+ * **A note also cites a word its frontmatter names whole**: a full name or an
+ * alias, or an affiliation's rank or office title, that spells the word's form
+ * or a variant under the same folding. A word found only inside a longer name
+ * is no citation.
+ *
+ * A language note never cites its own words, and its own body cites nothing:
+ * its italics are the words it describes. The index carries what this derives as each language record's
+ * `citations` — the citing notes' Addresses, never their text — and the
+ * `attested` column of the `words` relation and the `usage` of a `dictionary`
+ * entry read it.
+ *
+ * @module
+ */
+
+import MarkdownIt from "markdown-it";
+
+import { isLanguageRecord, noteAddressOf } from "./word-rows.mjs";
+
+/** The parser marked uses are read with: CommonMark's emphasis and nothing added. */
+const PARSER = new MarkdownIt("commonmark");
+
+/** Code-point order, so a list is the same on every machine. */
+const compare = (a, b) =>
+    a < b ? -1
+    : a > b ? 1
+    : 0;
+
+/** Whether a value is a plain map. */
+const isMap = (value) => Boolean(value) && typeof value === "object" && !Array.isArray(value);
+
+/**
+ * A form as uses are matched against it: diacritics taken off, case folded.
+ *
+ * @param {unknown} text - A form or a use.
+ * @returns {string} The folded form.
+ */
+export function foldForm(text) {
+    return String(text ?? "")
+        .normalize("NFD")
+        .replace(/\p{M}/gu, "")
+        .toLowerCase()
+        .normalize("NFC")
+        .trim();
+}
+
+/**
+ * The marked uses in a note's body: the text of every italic span that holds
+ * text alone, in document order.
+ *
+ * @param {string} body - The note's Markdown body.
+ * @returns {string[]} Each span's text.
+ */
+export function markedUses(body) {
+    const text = String(body ?? "");
+    if (!/[_*]/.test(text)) return [];
+    const uses = [];
+    for (const token of PARSER.parse(text, {})) {
+        if (token.type !== "inline" || !token.children) continue;
+        let span;
+        for (const child of token.children) {
+            if (child.type === "em_open") span = [];
+            else if (child.type === "em_close") {
+                if (span) {
+                    const use = span.join("").trim();
+                    if (use && !use.includes("[[")) uses.push(use);
+                }
+                span = undefined;
+            } else if (span) {
+                if (child.type === "text") span.push(child.content);
+                else span = undefined;
+            }
+        }
+    }
+    return uses;
+}
+
+/**
+ * The names a note carries whole, which cite a word they spell exactly: its
+ * full name and aliases, and an affiliation's rank and office titles.
+ *
+ * @param {Record<string, any>} record - An index record.
+ * @returns {string[]}
+ */
+function namesCarried(record) {
+    const out = [];
+    const name = isMap(record.name) ? record.name : {};
+    if (typeof name.full === "string") out.push(name.full);
+    for (const alias of Array.isArray(name.aliases) ? name.aliases : [])
+        if (typeof alias === "string") out.push(alias);
+    const governance = isMap(record.data?.governance) ? record.data.governance : {};
+    for (const rank of Array.isArray(governance.ranks) ? governance.ranks : [])
+        if (isMap(rank) && typeof rank.title === "string") out.push(rank.title);
+    if (isMap(governance.offices)) out.push(...Object.keys(governance.offices));
+    return out;
+}
+
+/**
+ * Each language note's cited words: for every word some note's marked use
+ * spells, the Addresses of the notes citing it.
+ *
+ * @param {Iterable<Record<string, any>>} records - Index records.
+ * @param {Iterable<{record: Record<string, any>, body: string}>} notes - The
+ *   notes whose bodies are read, each with its record.
+ * @returns {Map<string, Map<string, string[]>>} By language Address, by the
+ *   word's form, the citing notes in code-point order.
+ */
+export function wordCitations(records, notes) {
+    const byFold = new Map();
+    for (const record of records) {
+        if (!isLanguageRecord(record)) continue;
+        const language = noteAddressOf(record);
+        if (!language) continue;
+        for (const word of Array.isArray(record.data?.words) ? record.data.words : []) {
+            if (!isMap(word) || typeof word.form !== "string") continue;
+            const spellings = [
+                word.form,
+                ...(Array.isArray(word.variants) ? word.variants : []).filter(
+                    (variant) => typeof variant === "string",
+                ),
+            ];
+            for (const spelling of spellings) {
+                const fold = foldForm(spelling);
+                if (!fold) continue;
+                if (!byFold.has(fold)) byFold.set(fold, []);
+                byFold.get(fold).push({ language, form: word.form });
+            }
+        }
+    }
+    const cited = new Map();
+    if (!byFold.size) return cited;
+    /** Cite every word a folded spelling matches from `note`, never from its own language. */
+    const cite = (note, fold) => {
+        for (const { language, form } of byFold.get(fold) ?? []) {
+            if (language === note) continue;
+            if (!cited.has(language)) cited.set(language, new Map());
+            const words = cited.get(language);
+            if (!words.has(form)) words.set(form, new Set());
+            words.get(form).add(note);
+        }
+    };
+    for (const { record, body } of notes) {
+        if (isLanguageRecord(record)) continue;
+        const note = noteAddressOf(record);
+        if (!note) continue;
+        for (const fold of new Set(markedUses(body).map(foldForm))) cite(note, fold);
+    }
+    for (const record of records) {
+        if (record?.documents || record?.type === "folder") continue;
+        const note = noteAddressOf(record);
+        if (!note) continue;
+        for (const name of namesCarried(record)) cite(note, foldForm(name));
+    }
+    const out = new Map();
+    for (const [language, words] of cited)
+        out.set(language, new Map([...words].map(([form, set]) => [form, [...set].sort(compare)])));
+    return out;
+}
+
+/**
+ * Set each language record's `citations` from {@link wordCitations}: one entry
+ * per cited word, in the order the words stand, `{ form, notes }`. A language
+ * with no cited word carries no `citations`.
+ *
+ * @param {Record<string, any>[]} records - Index records, written in place.
+ * @param {Iterable<{record: Record<string, any>, body: string}>} notes - The
+ *   notes whose bodies are read.
+ * @returns {void}
+ */
+export function attachCitations(records, notes) {
+    const cited = wordCitations(records, notes);
+    for (const record of records) {
+        if (!isLanguageRecord(record)) continue;
+        const words = cited.get(noteAddressOf(record) ?? "");
+        if (!words) continue;
+        const citations = [];
+        for (const word of record.data.words)
+            if (isMap(word) && words.has(word.form))
+                citations.push({ form: word.form, notes: words.get(word.form) });
+        record.citations = citations;
+    }
+}