@heroiclands/package-build 23.8.0 → 23.10.0

package/engine/content-spans.mjs

@@ -17,6 +17,7 @@ import { slugify } from "./content-slug.mjs";
 import { scanPoetry } from "./content-poetry.mjs";
 import footnotePlugin from "markdown-it-footnote";
 import { parseExtensionAttributes, refusedAttributes } from "./extension-attributes.mjs";
+import { inlineText } from "./name-markup.mjs";
 
 /** Add bracket spans after recognized Markdown links and footnotes.
  * @param {import("markdown-it")} md - Markdown parser to extend.
@@ -57,7 +58,9 @@ export function spanMarkdownPlugin(md) {
             if (close === state.posMax)
                 parsed.problems.push("span attributes need closing {…} braces");
             else {
-                parsed = parseExtensionAttributes(state.src.slice(finish + 1, close));
+                parsed = parseExtensionAttributes(state.src.slice(finish + 1, close), {
+                    positions: true,
+                });
                 finish = close + 1;
             }
         }
@@ -73,6 +76,17 @@ export function spanMarkdownPlugin(md) {
             end: finish,
             text: state.src.slice(start + 1, end),
             problems,
+            attributePositions: Object.fromEntries(
+                Object.entries(parsed.positions ?? {}).map(([key, positions]) => [
+                    key,
+                    Object.fromEntries(
+                        Object.entries(positions).map(([part, range]) => [
+                            part,
+                            { start: range.start + end + 2, end: range.end + end + 2 },
+                        ]),
+                    ),
+                ]),
+            ),
         };
         if (parsed.id) token.attrSet("id", slugify(parsed.id));
         if (parsed.classes.length) token.attrSet("class", parsed.classes.join(" "));
@@ -84,6 +98,11 @@ export function spanMarkdownPlugin(md) {
             if (child.meta?.span) {
                 child.meta.start += start + 1;
                 child.meta.end += start + 1;
+                for (const positions of Object.values(child.meta.attributePositions ?? {}))
+                    for (const range of Object.values(positions)) {
+                        range.start += start + 1;
+                        range.end += start + 1;
+                    }
             }
         state.tokens.push(...nested);
         state.push("span_close", "span", -1);
@@ -98,6 +117,7 @@ const parser = new MarkdownIt({ html: true })
     .use(spanMarkdownPlugin);
 /** Read span anchors and located attribute diagnostics outside literal code.
  * @param {string} source - Authored Markdown.
+ * @returns {import("./name-markup.mjs").SpanScan} Actual spans, explicit names and located diagnostics.
  */
 export function scanSpans(source) {
     const text = String(source ?? ""),
@@ -112,14 +132,28 @@ export function scanSpans(source) {
         starts.push(offset);
         offset += line.length + 1;
     }
+    const located = (start, end) => {
+        let lo = 0,
+            hi = starts.length;
+        while (lo + 1 < hi) {
+            const mid = (lo + hi) >>> 1;
+            if (starts[mid] <= start) lo = mid;
+            else hi = mid;
+        }
+        return { start, end, line: lo + 1, column: start - starts[lo] + 1 };
+    };
     const collect = (children, position) => {
-        for (const child of children ?? []) {
+        for (const [index, child] of (children ?? []).entries()) {
             if (!child.meta?.span) continue;
             const start = position(child.meta.start),
                 end = position(child.meta.end);
-            const prefix = text.slice(0, start),
-                line = prefix.split("\n").length,
-                column = start - prefix.lastIndexOf("\n");
+            const { line, column } = located(start, end);
+            let depth = 1,
+                close = index + 1;
+            for (; close < children.length; close++) {
+                if (children[close].type === "span_open") depth++;
+                if (children[close].type === "span_close" && --depth === 0) break;
+            }
             spans.push({
                 id: child.attrGet("id") ?? "",
                 line,
@@ -127,13 +161,50 @@ export function scanSpans(source) {
                 start,
                 end,
                 text: child.meta.text,
+                contentEnd: position(child.meta.start + 1 + child.meta.text.length),
+                visible: inlineText(children.slice(index + 1, close)),
                 attrs: child.attrs ?? [],
+                attributePositions: Object.fromEntries(
+                    Object.entries(child.meta.attributePositions ?? {}).map(([key, positions]) => [
+                        key,
+                        Object.fromEntries(
+                            Object.entries(positions).map(([part, range]) => [
+                                part,
+                                located(position(range.start), position(range.end)),
+                            ]),
+                        ),
+                    ]),
+                ),
             });
             for (const message of child.meta.problems) errors.push({ line, column, message });
         }
     };
+    let row;
     for (const token of tokens) {
-        if (token.type !== "inline" || !token.map) continue;
+        if (token.type === "tr_open" && token.map) {
+            const line = lines[token.map[0]],
+                offsets = [];
+            let content = "";
+            // The existing table rule removes a backslash immediately before
+            // a pipe. Retain that projection's original character positions;
+            // actual row/cell tokens still own all table parsing and trimming.
+            for (let i = 0; i < line.length; i++) {
+                if (line[i] === "\\" && line[i + 1] === "|") continue;
+                offsets.push(i);
+                content += line[i];
+            }
+            offsets.push(line.length);
+            row = { content, offsets, line: token.map[0], cursor: 0 };
+        } else if (token.type === "tr_close") row = undefined;
+        if (token.type !== "inline") continue;
+        if (!token.map) {
+            if (!row) continue;
+            const column = row.content.indexOf(token.content, row.cursor);
+            if (column < 0) continue;
+            row.cursor = column + token.content.length;
+            collect(token.children, (pos) => starts[row.line] + row.offsets[column + pos]);
+            continue;
+        }
         const contentLines = token.content.split("\n"),
             positions = [];
         let local = 0;
@@ -159,7 +230,15 @@ export function scanSpans(source) {
     }
     spans.sort((a, b) => a.start - b.start);
     errors.sort((a, b) => a.line - b.line || a.column - b.column);
-    return { spans, errors };
+    return {
+        spans,
+        names: spans.filter((span) =>
+            span.attrs.some(
+                ([key, value]) => key === "class" && value.split(/\s+/).includes("name"),
+            ),
+        ),
+        errors,
+    };
 }
 
 /** Emit span tags while leaving their inline content as Markdown for Hugo.
@@ -182,7 +261,7 @@ export function renderSpans(source) {
         const attrs = span.attrs.map(([key, value]) => ` ${key}="${escape(value)}"`).join("");
         markdown =
             markdown.slice(0, span.start) +
-            `<span${attrs}>${renderSpans(span.text).markdown}</span>` +
+            `<span${attrs}>${renderSpans(markdown.slice(span.start + 1, span.contentEnd)).markdown}</span>` +
             markdown.slice(span.end);
     }
     return { markdown, errors };

package/engine/content-tables.mjs

@@ -14,6 +14,7 @@
 import { FENCE_LINE } from "./code-fences.mjs";
 import { PAGE_LIST_LANGUAGE } from "./page-lists.mjs";
 import { appendGeneratedSections } from "./generated-sections.mjs";
+import { insertLexicalSections } from "./lexical-sections.mjs";
 
 /**
  * Replace prepared SQL fences and page lists with Markdown while retaining
@@ -29,7 +30,8 @@ import { appendGeneratedSections } from "./generated-sections.mjs";
  *   Prepared results.
  * A note's generated sections — see {@link module:engine/generated-sections} —
  * are appended to the body first, from the `generated` text the preparation
- * set on its results, and expanded with it.
+ * set on its results, and expanded with it. Automatic lexical lists, carried
+ * on `lexical`, then fill their authored sections or append a new section.
  *
  * @returns {{markdown: string, errors: object[], warnings: object[], lineMap: object[],
  *   generatedFrom?: number}} `generatedFrom` is the 0-based line of the
@@ -148,10 +150,13 @@ export function expandContentTables(markdown, { source = "", sqlTables, pageList
         i = close;
     }
     return {
-        markdown: out.join("\n"),
+        ...insertLexicalSections(out.join("\n"), sqlTables?.lexical, {
+            authored,
+            lineMap,
+            generatedFrom,
+            syntax: sqlTables?.syntax,
+        }),
         errors,
         warnings,
-        lineMap,
-        ...(generatedFrom === undefined ? {} : { generatedFrom }),
     };
 }

package/engine/content-tasklists.mjs

@@ -43,9 +43,10 @@
 
 import fs from "node:fs";
 import path from "node:path";
+import { capturedSources } from "./content-sources.mjs";
 
 import { matchAllOutsideCode } from "./code-fences.mjs";
-import { positionInBody } from "./diagnostics.mjs";
+import { positionInBody, splitFrontmatter } from "./diagnostics.mjs";
 
 /**
  * A list item's marker, immediately followed by `[ ]` or `[x]` and a space —
@@ -114,15 +115,30 @@ export function checkTaskLists(body, file, { bodyLine = 1, bodyColumn = 1 } = {}
  * @param {object} [opts]
  * @param {readonly string[]} [opts.skipDirectories] - Directory names to ignore
  *   in addition to the dot-directories always skipped.
+ * @param {Map<string, {raw: string}>} [opts.sources] - Exact captured source strings; omitted, walks and reads the tree.
  * @returns {{findings: Array<{file: string, line: number, column: number,
  *   severity: "warning", message: string}>, files: number}} The findings, and
  *   how many files were read.
  */
-export function lintContentTaskLists(contentBase, { skipDirectories = [] } = {}) {
+export function lintContentTaskLists(contentBase, { skipDirectories = [], sources } = {}) {
     const skip = new Set(skipDirectories);
     /** @type {Array<{file: string, line: number, column: number, severity: "warning", message: string}>} */
     const findings = [];
     let files = 0;
+    if (sources) {
+        for (const [full, content] of capturedSources(
+            contentBase,
+            sources,
+            skipDirectories,
+            new Set([".md", ".markdown"]),
+        )) {
+            files++;
+            findings.push(
+                ...checkTaskLists(...bodyOf(content, path.relative(process.cwd(), full))),
+            );
+        }
+        return { findings, files };
+    }
 
     /** @param {string} dir - Directory to descend into. */
     const walk = (dir) => {
@@ -171,10 +187,10 @@ export function lintContentTaskLists(contentBase, { skipDirectories = [] } = {})
  *   arguments {@link checkTaskLists} takes.
  */
 function bodyOf(content, file) {
-    const match = content.match(/^---\n([\s\S]*?)\n---\n?([\s\S]*)$/);
+    const match = splitFrontmatter(content);
     if (!match) return [content, file, { bodyLine: 1, bodyColumn: 1 }];
 
-    const raw = match[2];
+    const raw = match.body;
     const body = raw.trim();
     const bodyStart = content.length - raw.length + (raw.length - raw.trimStart().length);
     const before = content.slice(0, bodyStart);

package/engine/culture-reach.mjs

@@ -15,17 +15,14 @@
  * Which terms reach a culture or an affiliation, and whether an affiliation
  * reaches a culture at all.
  *
- * A culture's terms are the words of its language, and the ranks, offices and
- * language words of every affiliation that reaches it — through its own
- * `data.culture`, the one the index resolves for it, or the culture of an
- * affiliation above it through `data.parents` —
- * each affiliation's language being the one its glossary resolves to
- * ({@link module:engine/glossary-language}). An affiliation's terms are its own
- * ranks and offices and the words of that language. A language's words include
- * the words it inherits through `data.family`, and a word is a term only where
+ * A culture can name the ranks and offices of every affiliation that reaches
+ * it through `data.culture` or `data.parents`. An affiliation can name its own
+ * ranks and offices. Each glossary entry can also name a word of its resolved
+ * language ({@link module:engine/glossary-language}). A language's words include
+ * those it inherits through `data.family`, and a word is a term only where
  * its class's base is a glossary base. The `dictionary` relation
- * ({@link module:engine/dictionary-rows}) names a glossary entry's term from
- * the same reach.
+ * ({@link module:engine/dictionary-rows}) uses the same resolved-language and
+ * governance sources.
  *
  * **An affiliation that reaches no culture is a warning**, at its `type`, in a
  * tree that describes peoples, which is a tree holding at least one culture
@@ -42,7 +39,7 @@ import { AddressLink } from "./address-values.mjs";
 import { positionOfFrontmatterPath } from "./diagnostics.mjs";
 import { glossaryLanguage } from "./glossary-language.mjs";
 import { parseWordClass, WORD_CLASS_BASES } from "./word-classes.mjs";
-import { LANGUAGE_SUBTYPE } from "./word-terms.mjs";
+import { glossSenses, LANGUAGE_SUBTYPE } from "./word-terms.mjs";
 
 /** The bases whose words are glossary terms. */
 const GLOSSARY_BASES = new Set(
@@ -189,6 +186,11 @@ export function entryLanguage(index, note, entry) {
  * @property {Map<string, Map<string, string[]>>} terms - Each culture's and
  *   affiliation's terms by NFC name, each with the sense texts its sources
  *   give it.
+ * @property {Map<string, Map<string, string[]>>} governance - The ranks and
+ *   offices each owner can name.
+ * @property {Map<string, Array<{form: string, gloss: unknown}>>} words - Each
+ *   language's dictionary words.
+ * @property {Map<string, string|undefined>} families - Each language's parent.
  */
 
 /** @type {WeakMap<object, GlossaryCorpus>} */
@@ -239,6 +241,23 @@ function reachedCultures(index, note) {
     return [...out];
 }
 
+/** Add the nearest dictionary definition of each form in a language's lineage. */
+function addLanguageWords(terms, key, language, words, families) {
+    const seen = new Set();
+    const forms = new Set();
+    for (let current = language; current && !seen.has(current); current = families.get(current)) {
+        seen.add(current);
+        for (const word of words.get(current) ?? []) {
+            const form = glossaryName(word.form);
+            if (forms.has(form)) continue;
+            forms.add(form);
+            const senses = glossSenses(word.gloss);
+            if (!senses.length) addTerm(terms, key, form, undefined);
+            for (const sense of senses) addTerm(terms, key, form, sense);
+        }
+    }
+}
+
 /**
  * The glossary corpus of an index.
  *
@@ -257,18 +276,7 @@ export function glossaryCorpus(index) {
             if (key) families.set(key, addressKeyOf(dataOf(note).family, index, "skill"));
         }
     const terms = new Map();
-    // A language's words include those it inherits through `data.family`.
-    const addWords = (key, language) => {
-        const seen = new Set();
-        for (
-            let current = language;
-            current && !seen.has(current);
-            current = families.get(current)
-        ) {
-            seen.add(current);
-            for (const word of words.get(current) ?? []) addTerm(terms, key, word.form, word.gloss);
-        }
-    };
+    const governance = new Map();
     for (const note of notes) {
         const culture = isCultureNote(note);
         if (!culture && typeOf(note) !== "affiliation") continue;
@@ -276,18 +284,22 @@ export function glossaryCorpus(index) {
         if (!key) continue;
         if (!terms.has(key)) terms.set(key, new Map());
         const data = dataOf(note);
-        if (culture) {
-            addWords(key, addressKeyOf(data.language, index, "skill"));
-            continue;
-        }
+        addLanguageWords(terms, key, entryLanguage(index, note, undefined), words, families);
+        if (culture) continue;
         addGovernance(terms, key, data);
-        addWords(key, entryLanguage(index, note, undefined));
+        addGovernance(governance, key, data);
         for (const home of reachedCultures(index, note)) {
             addGovernance(terms, home, data);
-            addWords(home, entryLanguage(index, note, undefined));
+            addGovernance(governance, home, data);
         }
     }
-    const corpus = { describesPeoples: notes.some(isCultureNote), terms };
+    const corpus = {
+        describesPeoples: notes.some(isCultureNote),
+        terms,
+        governance,
+        words,
+        families,
+    };
     CORPUS.set(index, corpus);
     return corpus;
 }
@@ -298,16 +310,31 @@ export function glossaryCorpus(index) {
  *
  * @param {object} index - The link index.
  * @param {object} note - The culture or affiliation note.
+ * @param {object} [entry] - A glossary entry, whose resolved language selects
+ *   its dictionary words. Absent, the note's own language is used.
  * @returns {Map<string, string[]>} The terms; empty for a note no source reaches.
  */
-export function noteTerms(index, note) {
+export function noteTerms(index, note, entry) {
     const key = ownKey(note, index);
-    return (key && glossaryCorpus(index).terms.get(key)) || new Map();
+    const corpus = glossaryCorpus(index);
+    if (!entry) return (key && corpus.terms.get(key)) || new Map();
+    const own = new Map(
+        [...(corpus.governance.get(key) ?? [])].map(([name, senses]) => [name, [...senses]]),
+    );
+    addLanguageWords(
+        new Map([[key, own]]),
+        key,
+        entryLanguage(index, note, entry),
+        corpus.words,
+        corpus.families,
+    );
+    return own;
 }
 
 /**
  * Warn of an affiliation that reaches no culture, its own or one it inherits,
- * in a tree that describes peoples.
+ * in a tree that describes peoples. Its ranks and offices then have no
+ * culture associations for term queries.
  *
  * @param {object} note - An `affiliation` note.
  * @param {{index?: object}} [context]
@@ -323,7 +350,7 @@ export function checkAffiliationReach(note, { index } = {}) {
             severity: "warning",
             message:
                 "this affiliation reaches no culture, its own or one inherited through " +
-                "`data.parents`, so its ranks, offices and language enter no culture's glossary; " +
+                "`data.parents`, so its ranks and offices have no culture associations for term queries; " +
                 "state `data.culture`",
         },
     ];

package/engine/data-keys.mjs

@@ -70,6 +70,8 @@ import { nearest } from "./near-miss.mjs";
  *   every surface shows it exactly as written, never recased.
  * @property {string} shape - The shape in words, for a finding and the reference.
  * @property {boolean} [required] - The key must be stated, with a value.
+ * @property {boolean} [allowEmpty] - A required string may be empty where
+ *   empty text is a stated value.
  * @property {boolean} [nullable] - An explicit `null` states the key, for a
  *   required key whose null has a meaning of its own.
  * @property {string} [label] - The infobox row label, where humanising the key
@@ -234,7 +236,7 @@ export function pathLabel(path) {
 function stated(value, spec) {
     if (value === undefined) return false;
     if (value === null) return Boolean(spec.nullable);
-    if (typeof value === "string") return value.trim() !== "";
+    if (typeof value === "string") return Boolean(spec.allowEmpty) || value.trim() !== "";
     return true;
 }
 

package/engine/diagnostics.mjs

@@ -46,6 +46,17 @@ import fs from "node:fs";
 import path from "node:path";
 import YAML, { LineCounter } from "yaml";
 
+/**
+ * Split opening YAML frontmatter from its untrimmed body without parsing it.
+ * LF and CRLF delimiters retain the exact source strings and their offsets.
+ * @param {string} raw - Complete Markdown source.
+ * @returns {{yaml: string, body: string}|null} Source slices, or no fence.
+ */
+export function splitFrontmatter(raw) {
+    const match = raw.match(/^---\r?\n([\s\S]*?)\r?\n---(?:\r?\n)?([\s\S]*)$/);
+    return match ? { yaml: match[1], body: match[2] } : null;
+}
+
 /**
  * The `file:line:column` locator, with whatever is known.
  *
@@ -201,9 +212,9 @@ export function positionInBody(body, offset, { bodyLine = 1, bodyColumn = 1, lin
  */
 export function positionInFrontmatter(raw, key, value = undefined, { topLevel = false } = {}) {
     if (typeof raw !== "string" || !key) return {};
-    const fence = raw.match(/^---\n([\s\S]*?)\n---/);
+    const fence = splitFrontmatter(raw);
     if (!fence) return {};
-    const block = fence[1];
+    const block = fence.yaml;
     const lines = block.split("\n");
 
     // A value locates the exact entry; the key alone locates its declaration.
@@ -437,10 +448,10 @@ export function yamlKeyPath(field) {
 export function frontmatterValueAt(raw, keyPath) {
     if (typeof raw !== "string" || !Array.isArray(keyPath) || keyPath.length === 0)
         return undefined;
-    const fence = raw.match(/^---\n([\s\S]*?)\n---/);
+    const fence = splitFrontmatter(raw);
     if (!fence) return undefined;
     try {
-        const node = YAML.parseDocument(fence[1]).getIn(keyPath, true);
+        const node = YAML.parseDocument(fence.yaml).getIn(keyPath, true);
         const value = node?.value;
         return ["string", "number", "boolean"].includes(typeof value) ? value : undefined;
     } catch {
@@ -473,9 +484,9 @@ export function frontmatterValueAt(raw, keyPath) {
  */
 export function positionOfFrontmatterPath(raw, keyPath, { key = false } = {}) {
     if (typeof raw !== "string") return {};
-    const fence = raw.match(/^---\n([\s\S]*?)\n---/);
+    const fence = splitFrontmatter(raw);
     if (!fence) return {};
-    const position = positionOfYamlPath(fence[1], keyPath, { key });
+    const position = positionOfYamlPath(fence.yaml, keyPath, { key });
     // +1: the fence's line 1 is the file's line 2, the opening `---` being the
     // first. `positionOfYamlPath` already counts from 1 within the block.
     return position.line === undefined ? {} : { ...position, line: position.line + 1 };