@heroiclands/package-build 25.5.0 → 25.5.2

package/CHANGELOG.md

@@ -1,5 +1,17 @@
 # @heroiclands/package-build
 
+## 25.5.2
+
+**Languages and scripts**
+
+- A sentence that runs across several spans and links in one language is checked as one sentence, so the period that ends it is not reported as opening the next.
+
+## 25.5.1
+
+**Builds and commands**
+
+- `package-build markdown` checks a repository of any size, rather than crashing on large ones.
+
 ## 25.5.0
 
 **Languages and scripts**

package/docs/api.md

@@ -1449,12 +1449,14 @@ The charset authored content is held to, so a book can choose its face. The pack
 Exact permitted Roman source characters for configured languages, before script
 encoding. Inventories come from `data.roman` on each language's own note.
 
-| Export                   | Signature                                | Returns                       | Use it when                                                                                   |
-| ------------------------ | ---------------------------------------- | ----------------------------- | --------------------------------------------------------------------------------------------- |
-| `isRomanWhitespace`      | `isRomanWhitespace(char)`                | `boolean`                     | recognizing standard spaces and ASCII tab or line whitespace                                  |
-| `romanInventoryProblems` | `romanInventoryProblems(value)`          | `string[]`                    | checking NFC, repeated characters and forbidden whitespace or controls in a literal inventory |
-| `romanInventoriesOf`     | `romanInventoriesOf(records, languages)` | `Map<string, RomanInventory>` | looking up a configured language's own inventory by name or BCP 47 tag                        |
-| `romanTextProblems`      | `romanTextProblems(text, inventory)`     | `{offset, message}[]`         | validating standard Roman punctuation and language additions with exact UTF-16 offsets        |
+| Export                    | Signature                                    | Returns                       | Use it when                                                                                                                                                          |
+| ------------------------- | -------------------------------------------- | ----------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| `isRomanWhitespace`       | `isRomanWhitespace(char)`                    | `boolean`                     | recognizing standard spaces and ASCII tab or line whitespace                                                                                                         |
+| `romanInventoryProblems`  | `romanInventoryProblems(value)`              | `string[]`                    | checking NFC, repeated characters and forbidden whitespace or controls in a literal inventory                                                                        |
+| `romanInventoriesOf`      | `romanInventoriesOf(records, languages)`     | `Map<string, RomanInventory>` | looking up a configured language's own inventory by name or BCP 47 tag                                                                                               |
+| `romanTextProblems`       | `romanTextProblems(text, inventory, stream)` | `{offset, message}[]`         | validating standard Roman punctuation and language additions with exact UTF-16 offsets; pass a `stream` state to continue sentences and quotations from earlier runs |
+| `romanPunctuationState`   | `romanPunctuationState()`                    | `{sentenceStart, quotes}`     | starting a stream of runs whose sentences and quotations continue from one run to the next                                                                           |
+| `romanUnclosedQuotations` | `romanUnclosedQuotations(quotes)`            | `{offset, message}[]`         | reporting the quotations still open where a stream of runs ends                                                                                                      |
 
 ### `engine.pathCharset`
 

package/engine/content-scripts.mjs

@@ -63,7 +63,12 @@ import { unknownWordMessage, unknownWords, wordBooksFromRecords } from "./run-wo
 import { metreBooksFromRecords, metreFindings, metresFromRecords } from "./verse-metre.mjs";
 import { encodeWithLanguage } from "./encoding-language.mjs";
 import { encodingsFromRecords, encodingAddress } from "./encoding-notes.mjs";
-import { romanInventoriesOf, romanTextProblems } from "./roman-characters.mjs";
+import {
+    romanInventoriesOf,
+    romanPunctuationState,
+    romanTextProblems,
+    romanUnclosedQuotations,
+} from "./roman-characters.mjs";
 import { plainRomanRun } from "./roman-run.mjs";
 import { plainRomanVerse } from "./roman-verse.mjs";
 import { encodeText } from "./encoding.mjs";
@@ -1175,6 +1180,7 @@ function analyseRuns(source, ctx, syntax) {
             rawReferences.push({
                 lang: keys.lang,
                 font: keys.font,
+                enc: keys.enc,
                 from: match.index + 2 + bar + 1,
                 to: match.index + close - 1,
                 where: construct.where,
@@ -1363,24 +1369,6 @@ function analyseRuns(source, ctx, syntax) {
             (ctx.romanInventories?.get(lang) ?? ctx.romanInventories?.get(lang.toLowerCase()))
         );
 
-    /** Check visible source before selection or lexicon-native replacement. */
-    const roman = (parts, detail) => {
-        const inventory = inventoryOf(detail.run.lang);
-        if (!inventory) return;
-        if (!inventory.characters) {
-            report(
-                { line: detail.run.line, column: detail.run.column },
-                `the language note for \`${inventory.language}\` must declare \`data.roman\``,
-            );
-            return;
-        }
-        for (const [from, to] of parts) {
-            const plain = plainRomanRun(text, { from, to });
-            for (const problem of romanTextProblems(plain.text, inventory))
-                report(positionOf(starts, plain.offsets[problem.offset] ?? from), problem.message);
-        }
-    };
-
     /**
      * Give a scroll fence the face it is set in, and report a fence that names
      * none or whose script is written in lines. A fence that names no face
@@ -1646,7 +1634,128 @@ function analyseRuns(source, ctx, syntax) {
             .filter((detail) => detail.run.kind === "span")
             .map((detail) => [detail.run.start, detail]),
     );
-    for (const owner of details.filter((detail) => ownerOf(detail) === detail)) {
+    // Inline runs, spans and labelled links, of one resolved language and
+    // encoding separated only by whitespace form one punctuation stream: a
+    // sentence or a quotation may run through several of them. Each run opens
+    // in the state its predecessor in the stream closed in, and quotations
+    // still open are reported once, where the stream ends. A block (a div,
+    // verse or scroll) is a stream of its own and never joins an inline run,
+    // as a paragraph does not continue a sentence from outside it; it still
+    // separates the runs on either side of it.
+    const ownerDetails = details
+        .filter((detail) => ownerOf(detail) === detail)
+        .sort((a, b) => a.run.start - b.run.start);
+    const streamEntry = (where, lang, enc, checkable, inline) => {
+        const inventory = inventoryOf(lang);
+        // A block's bounds are its opening and closing line indices.
+        const end = Math.min(where.end, lines.length - 1);
+        return {
+            inline,
+            start: inline ? where.start : starts[where.start],
+            end: inline ? where.end : starts[end] + lines[end].length,
+            line: where.line,
+            column: where.column,
+            lang,
+            language: tagOf(lang)?.toLowerCase() ?? lang,
+            enc,
+            inventory,
+            checked: checkable && Boolean(inventory?.characters),
+            previous: undefined,
+            continued: false,
+            state: undefined,
+            problems: undefined,
+        };
+    };
+    const streamRuns = [
+        ...ownerDetails.map((detail) => ({
+            ...streamEntry(
+                detail.run,
+                detail.run.lang,
+                detail.run.enc,
+                Boolean(boundsOf(detail) && detail.checkRoman),
+                detail.run.kind === "span",
+            ),
+            detail,
+        })),
+        ...rawReferences.map((reference) => ({
+            ...streamEntry(reference.where, reference.lang, reference.enc, true, true),
+            reference,
+        })),
+    ].sort((a, b) => a.start - b.start);
+    for (const entry of streamRuns) {
+        if (!entry.checked || !entry.inline) continue;
+        const previous = streamRuns
+            .filter((other) => other.end <= entry.start)
+            .reduce((last, other) => (last && last.end >= other.end ? last : other), undefined);
+        if (
+            previous?.checked &&
+            previous.inline &&
+            previous.inventory === entry.inventory &&
+            previous.language === entry.language &&
+            previous.enc === entry.enc &&
+            /^[\p{Zs}\t\r\n]*$/u.test(text.slice(previous.end, entry.start))
+        ) {
+            entry.previous = previous;
+            previous.continued = true;
+        }
+    }
+    const streamRunOf = new Map(
+        streamRuns.filter((entry) => entry.detail).map((entry) => [entry.detail, entry]),
+    );
+    /** The punctuation state a run opens in, a copy of its predecessor's. */
+    const openingState = (entry) => {
+        const previous = entry.previous && closingState(entry.previous);
+        return previous ?
+                {
+                    sentenceStart: previous.sentenceStart,
+                    quotes: previous.quotes.map((quote) => ({ ...quote })),
+                }
+            :   romanPunctuationState();
+    };
+    /** A run's closing punctuation state, checking a labelled link on demand. */
+    const closingState = (entry) => {
+        if (entry.reference) checkReference(entry);
+        return entry.state;
+    };
+    /**
+     * Check a run's Roman text in its stream, giving findings by source offset.
+     *
+     * @param {any} entry - The run.
+     * @param {string} roman - Its projected Roman text.
+     * @param {(offset: number) => number} source - A text offset's source offset.
+     */
+    const checkStream = (entry, roman, source) => {
+        const state = openingState(entry);
+        const problems = romanTextProblems(roman, entry.inventory, state).map((problem) => ({
+            ...problem,
+            offset: source(problem.offset),
+        }));
+        for (const quote of state.quotes) quote.source ??= source(quote.offset);
+        if (!entry.continued)
+            problems.push(
+                ...romanUnclosedQuotations(
+                    state.quotes.map((quote) => ({ ...quote, offset: quote.source })),
+                ),
+            );
+        entry.state = state;
+        return problems;
+    };
+    /** Check a labelled link's Roman source once, in its stream. */
+    const checkReference = (entry) => {
+        if (entry.problems) return entry.problems;
+        entry.problems = [];
+        if (!entry.checked) return entry.problems;
+        const { reference } = entry;
+        const plain = plainRomanRun(text, { from: reference.from, to: reference.to });
+        entry.problems = checkStream(
+            entry,
+            plain.text,
+            (offset) => plain.offsets[offset] ?? reference.from,
+        );
+        return entry.problems;
+    };
+
+    for (const owner of ownerDetails) {
         const bounds = boundsOf(owner);
         if (!bounds || !owner.checkRoman) continue;
         const owned = details.filter((detail) => ownerOf(detail) === owner);
@@ -1703,12 +1812,15 @@ function analyseRuns(source, ctx, syntax) {
                 { line: owner.run.line, column: owner.run.column },
                 `the language note for \`${inventory.language}\` must declare \`data.roman\``,
             );
-        else if (inventory)
-            for (const problem of romanTextProblems(projection.text, inventory))
-                report(
-                    positionOf(starts, projection.offsets[problem.offset] ?? bounds[0]),
-                    problem.message,
-                );
+        else if (inventory) {
+            const entry = streamRunOf.get(owner);
+            for (const problem of checkStream(
+                entry,
+                projection.text,
+                (offset) => projection.offsets[offset] ?? bounds[0],
+            ))
+                report(positionOf(starts, problem.offset), problem.message);
+        }
         for (const problem of projection.errors ?? [])
             report(positionOf(starts, problem.offset), problem.message);
 
@@ -1830,16 +1942,15 @@ function analyseRuns(source, ctx, syntax) {
         for (const detail of owned) detail.plan = plan;
     }
 
-    for (const reference of rawReferences) {
-        const font = reference.font;
-        roman([[reference.from, reference.to]], {
-            run: {
-                lang: reference.lang,
-
-                ...reference.where,
-            },
-            face: font === undefined ? undefined : faces.get(font),
-        });
+    for (const entry of streamRuns.filter((entry) => entry.reference)) {
+        const inventory = inventoryOf(entry.lang);
+        if (inventory && !inventory.characters)
+            report(
+                { line: entry.line, column: entry.column },
+                `the language note for \`${inventory.language}\` must declare \`data.roman\``,
+            );
+        for (const problem of checkReference(entry))
+            report(positionOf(starts, problem.offset), problem.message);
     }
 
     for (const reference of independentReferences) {

package/engine/markdownlint-run.mjs

@@ -451,6 +451,9 @@ function gitignoreHooks(root) {
     };
 }
 
+/** Sources per markdownlint call: it lints `strings` by recursing once per entry, so a large set overflows the stack. */
+const LINT_BATCH_SIZE = 200;
+
 /**
  * Lint markdown under a root with the `markdownlint` library.
  *
@@ -560,10 +563,19 @@ export async function runMarkdownlint(params) {
         }
         const lintSources = async (selected) => {
             const { files: _files, ...shared } = options;
-            const results = await lint({
-                ...shared,
-                strings: Object.fromEntries(selected.map((file) => [file, sources.get(file).text])),
-            });
+            const results = {};
+            for (let start = 0; start < selected.length; start += LINT_BATCH_SIZE) {
+                const batch = selected.slice(start, start + LINT_BATCH_SIZE);
+                Object.assign(
+                    results,
+                    await lint({
+                        ...shared,
+                        strings: Object.fromEntries(
+                            batch.map((file) => [file, sources.get(file).text]),
+                        ),
+                    }),
+                );
+            }
             for (const file of selected) {
                 const protectedSource = sources.get(file);
                 results[file] = (results[file] ?? [])

package/engine/roman-characters.mjs

@@ -109,9 +109,14 @@ export function romanInventoriesOf(records, languages = {}) {
  * No punctuation, case or normalization repair is performed.
  * @param {string} text - Visible Roman source.
  * @param {RomanInventory} inventory - Its language's declared characters.
+ * @param {RomanPunctuationState} [stream] - The punctuation state of a stream
+ *   of runs the text continues, from {@link romanPunctuationState}; updated to
+ *   the state the text closes in. Its unclosed quotations are left for
+ *   {@link romanUnclosedQuotations} at the end of the stream. Without it the
+ *   text is a stream of its own and its unclosed quotations are reported.
  * @returns {Array<{offset: number, message: string}>}
  */
-export function romanTextProblems(text, inventory) {
+export function romanTextProblems(text, inventory, stream) {
     const out = [];
     if (!inventory.characters) return out;
     const decomposed = new Map(
@@ -119,8 +124,9 @@ export function romanTextProblems(text, inventory) {
             .filter(({ segment }) => segment !== segment.normalize("NFC"))
             .map(({ segment, index }) => [index, segment]),
     );
-    const quotes = [];
-    let sentenceStart = true;
+    const state = stream ?? romanPunctuationState();
+    const { quotes } = state;
+    let sentenceStart = state.sentenceStart;
     const sentenceText = () => {
         sentenceStart = false;
         if (quotes.length) quotes.at(-1).hasText = true;
@@ -245,10 +251,41 @@ export function romanTextProblems(text, inventory) {
         } else sentenceText();
         offset += char.length;
     }
-    for (const opening of quotes)
-        report(
-            opening.offset,
-            `quotation \`${opening.char}\` needs its closing \`${opening.char === "«" ? "»" : "›"}\``,
-        );
+    state.sentenceStart = sentenceStart;
+    if (!stream) out.push(...romanUnclosedQuotations(quotes));
     return out.sort((a, b) => a.offset - b.offset);
 }
+
+/**
+ * @typedef {object} RomanQuotation
+ * @property {string} char - The opening mark, `«` or `‹`.
+ * @property {number} offset - Its offset in the text that opened it.
+ * @property {boolean} sentenceStart - Whether its sentence had text before it.
+ * @property {boolean} hasText - Whether it holds sentence text.
+ */
+
+/**
+ * @typedef {object} RomanPunctuationState
+ * @property {boolean} sentenceStart - Whether no sentence text precedes the next mark.
+ * @property {RomanQuotation[]} quotes - Open quotations, outermost first.
+ */
+
+/**
+ * The punctuation state at the start of a stream of runs.
+ * @returns {RomanPunctuationState}
+ */
+export function romanPunctuationState() {
+    return { sentenceStart: true, quotes: [] };
+}
+
+/**
+ * Findings for quotations still open at the end of a stream.
+ * @param {ReadonlyArray<RomanQuotation>} quotes - The open quotations.
+ * @returns {Array<{offset: number, message: string}>} At each opening's `offset`.
+ */
+export function romanUnclosedQuotations(quotes) {
+    return quotes.map((opening) => ({
+        offset: opening.offset,
+        message: `quotation \`${opening.char}\` needs its closing \`${opening.char === "«" ? "»" : "›"}\``,
+    }));
+}

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@heroiclands/package-build",
-    "version": "25.5.0",
+    "version": "25.5.2",
     "description": "Shared toolchain for building and shipping a HeroicLands Foundry VTT package \u2014 content compilation, manifest, localization, staging, bundle, release and deployment.",
     "license": "GPL-3.0-or-later",
     "type": "module",

package/types/engine/roman-characters.d.mts

@@ -26,9 +26,40 @@ export function romanInventoriesOf(records: ReadonlyMap<string, Record<string, a
  * No punctuation, case or normalization repair is performed.
  * @param {string} text - Visible Roman source.
  * @param {RomanInventory} inventory - Its language's declared characters.
+ * @param {RomanPunctuationState} [stream] - The punctuation state of a stream
+ *   of runs the text continues, from {@link romanPunctuationState}; updated to
+ *   the state the text closes in. Its unclosed quotations are left for
+ *   {@link romanUnclosedQuotations} at the end of the stream. Without it the
+ *   text is a stream of its own and its unclosed quotations are reported.
  * @returns {Array<{offset: number, message: string}>}
  */
-export function romanTextProblems(text: string, inventory: RomanInventory): Array<{
+export function romanTextProblems(text: string, inventory: RomanInventory, stream?: RomanPunctuationState): Array<{
+    offset: number;
+    message: string;
+}>;
+/**
+ * @typedef {object} RomanQuotation
+ * @property {string} char - The opening mark, `«` or `‹`.
+ * @property {number} offset - Its offset in the text that opened it.
+ * @property {boolean} sentenceStart - Whether its sentence had text before it.
+ * @property {boolean} hasText - Whether it holds sentence text.
+ */
+/**
+ * @typedef {object} RomanPunctuationState
+ * @property {boolean} sentenceStart - Whether no sentence text precedes the next mark.
+ * @property {RomanQuotation[]} quotes - Open quotations, outermost first.
+ */
+/**
+ * The punctuation state at the start of a stream of runs.
+ * @returns {RomanPunctuationState}
+ */
+export function romanPunctuationState(): RomanPunctuationState;
+/**
+ * Findings for quotations still open at the end of a stream.
+ * @param {ReadonlyArray<RomanQuotation>} quotes - The open quotations.
+ * @returns {Array<{offset: number, message: string}>} At each opening's `offset`.
+ */
+export function romanUnclosedQuotations(quotes: ReadonlyArray<RomanQuotation>): Array<{
     offset: number;
     message: string;
 }>;
@@ -43,3 +74,31 @@ export type RomanInventory = {
      */
     characters: ReadonlySet<string> | undefined;
 };
+export type RomanQuotation = {
+    /**
+     * - The opening mark, `«` or `‹`.
+     */
+    char: string;
+    /**
+     * - Its offset in the text that opened it.
+     */
+    offset: number;
+    /**
+     * - Whether its sentence had text before it.
+     */
+    sentenceStart: boolean;
+    /**
+     * - Whether it holds sentence text.
+     */
+    hasText: boolean;
+};
+export type RomanPunctuationState = {
+    /**
+     * - Whether no sentence text precedes the next mark.
+     */
+    sentenceStart: boolean;
+    /**
+     * - Open quotations, outermost first.
+     */
+    quotes: RomanQuotation[];
+};