@heroiclands/package-build 23.13.0 → 24.0.1

package/engine/sql-tables.mjs

@@ -603,11 +603,7 @@ async function createEventRelation(connection, file, records, prefix, audience)
     fs.writeFileSync(file, rows.map((row) => JSON.stringify(row)).join("\n"));
     // `whenIn` and `untilIn` are keyed by calendar Address, and read as structs
     // whatever their number of keys, so `whenIn['<calendar>']` always binds.
-    await createTableFromFile(
-        connection,
-        `${prefix}events`,
-        readJsonAuto(file).replace(/\)$/, ", map_inference_threshold=-1)"),
-    );
+    await createTableFromFile(connection, `${prefix}events`, readJsonAuto(file));
 }
 
 /**
@@ -941,13 +937,20 @@ async function createRowRelation(connection, file, rows, view, columns, types) {
  * type's fields with `NULL` where a record lacks one. A dependency's index has
  * exactly the same shape and needs exactly the same reading.
  *
+ * `map_inference_threshold=-1` keeps every object a struct. Left at its default,
+ * DuckDB reads an object whose keys are sparsely spread across rows (the mean
+ * share of rows carrying a key falls under `field_appearance_threshold`) as a
+ * map, so `data.<key>` changes meaning with the corpus: one more field on one
+ * note can turn a plain comparison into a JSON conversion error.
+ *
  * @param {string} file - The JSONL to read.
  * @returns {string} The `SELECT … FROM read_json_auto(…)` clause.
  */
 function readJsonAuto(file) {
     return (
         `SELECT * FROM read_json_auto('${file.replace(/'/g, "''")}', ` +
-        `format='newline_delimited', union_by_name=true, maximum_object_size=20000000)`
+        `format='newline_delimited', union_by_name=true, maximum_object_size=20000000, ` +
+        `map_inference_threshold=-1)`
     );
 }
 

package/engine/text-maps.mjs

@@ -51,6 +51,14 @@
  *   `uncapitalised`, which a run reports; the capitalised word is then stated
  *   as a lexicon word of its own.
  *
+ * - **A logographic map** (`logographic: true`) writes every word, a word
+ *   being a match of the word pattern that holds a letter, by the writing a
+ *   lookup gives it (see {@link module:engine/logographic}), and its rows
+ *   write only the text between words: spaces, points, and anything else
+ *   with no letter. A word the lookup gives no writing is written as typed
+ *   and marked `unwritten`, which a run reports. A row of such a map whose
+ *   `from` holds a letter could never apply, and is refused.
+ *
  * `rows` is a list, never a keyed map, so `A` and `a` are two rows: the index
  * carries them as rows of the `maps` relation, never as struct keys.
  *
@@ -97,6 +105,8 @@ const BOUNDARY = /[\s\p{P}\p{S}]/u;
  *
  * @typedef {object} TextMap
  * @property {string} name - Its name.
+ * @property {boolean} logographic - Whether it writes words from the lexicon
+ *   alone, its rows only between them.
  * @property {ReadonlySet<string>} vowels - The characters `before: vowel` reads.
  * @property {readonly MapRow[]} rows - Its well-formed shared rows, in authored order.
  * @property {ReadonlyMap<string, readonly MapRow[]>} byFirst - Its shared rows
@@ -113,6 +123,8 @@ const BOUNDARY = /[\s\p{P}\p{S}]/u;
  *   tried first.
  * @property {(word: string) => string|undefined} [native] - A word's native
  *   form, for the word exactly as given, or `undefined` where it states none.
+ * @property {(word: string) => import("./logographic.mjs").Writing} [writing] -
+ *   A word's writing under a logographic map; without one, no word has one.
  */
 
 /**
@@ -236,6 +248,7 @@ export function compileMap(value) {
     }
     return Object.freeze({
         name: map.name,
+        logographic: map.logographic === true,
         vowels,
         rows,
         byFirst: indexRows(rows),
@@ -300,9 +313,9 @@ function rowAt(map, font, text, at, limit) {
  * @param {TextMap|undefined} map - The map.
  * @param {string} text - The text.
  * @param {string|undefined} font - The face.
- * @param {ReadonlyMap<number, {end: number, to: string, uncapitalised?: string}>} words -
- *   Words written by their native form, by offset.
- * @returns {{text: string, pieces: Array<{at: number, from: string, to: string, row: boolean, native?: boolean, uncapitalised?: string}>}}
+ * @param {ReadonlyMap<number, {end: number, to: string, uncapitalised?: string, unwritten?: object}>} words -
+ *   Words written whole, by offset.
+ * @returns {{text: string, pieces: Array<{at: number, from: string, to: string, row: boolean, native?: boolean, uncapitalised?: string, unwritten?: object}>}}
  *   As {@link applyMap}.
  */
 function applyRows(map, text, font, words) {
@@ -317,8 +330,9 @@ function applyRows(map, text, font, words) {
                 at,
                 from: text.slice(at, word.end),
                 to: word.to,
-                row: true,
-                native: true,
+                ...(word.unwritten === undefined ?
+                    { row: true, native: true }
+                :   { row: false, unwritten: word.unwritten }),
                 ...(word.uncapitalised === undefined ? {} : { uncapitalised: word.uncapitalised }),
             });
             out += word.to;
@@ -371,15 +385,31 @@ export function capitalNative(map, font, word, native) {
  *   as it is.
  * @param {string} text - The text, with no escapes in it.
  * @param {MapOptions} [options] - The face, and the native forms of words.
- * @returns {{text: string, pieces: Array<{at: number, from: string, to: string, row: boolean, native?: boolean}>}}
+ * @returns {{text: string, pieces: Array<{at: number, from: string, to: string, row: boolean, native?: boolean, unwritten?: object}>}}
  *   The rewritten text, and each piece of it with the UTF-16 offset in `text`
  *   it came from: a matched row's `from` and what it wrote, a word and its
  *   native form (`native: true`), or one character that passed through. A
  *   capitalised word whose native form the rows cannot capitalise is written
  *   as stated and carries `uncapitalised`, the lowercase form it was found by.
+ *   Under a logographic map, a word with no writing is written as typed and
+ *   carries `unwritten`, why it has none.
  */
-export function applyMap(map, text, { font, native } = {}) {
+export function applyMap(map, text, { font, native, writing } = {}) {
     const words = new Map();
+    if (map?.logographic) {
+        for (const match of text.matchAll(WORD)) {
+            const word = match[0];
+            if (!LETTER.test(word)) continue;
+            const found = writing ? writing(word) : { missing: true, noLexicon: true };
+            words.set(
+                match.index,
+                "text" in found ?
+                    { end: match.index + word.length, to: found.text }
+                :   { end: match.index + word.length, to: word, unwritten: found },
+            );
+        }
+        return applyRows(map, text, font, words);
+    }
     if (map && native)
         for (const match of text.matchAll(WORD)) {
             const word = match[0];
@@ -502,7 +532,8 @@ function namesIn(index) {
 /**
  * Check a note's `data.map`: its name, that no other note declares it, each
  * row written as a list, a `before` condition on a map with no `vowels`, an
- * empty `fonts`, and a generic family in it. The data-key check owns every
+ * empty `fonts`, a generic family in it, and a row of a logographic map that
+ * matches a letter. The data-key check owns every
  * key, its kind and the words a condition takes; whether a listed face is
  * installed, and draws what the map writes, is
  * {@link module:engine/map-lexicon.checkMapFonts}'s.
@@ -527,7 +558,9 @@ export function checkMapField(note, { index } = {}) {
         message,
     });
     const findings = [];
-    const { name, vowels, rows, fonts } = /** @type {Record<string, unknown>} */ (value);
+    const { name, vowels, rows, fonts, logographic } = /** @type {Record<string, unknown>} */ (
+        value
+    );
     if (typeof name === "string" && !MAP_NAME.test(name))
         findings.push(
             finding(
@@ -565,12 +598,26 @@ export function checkMapField(note, { index } = {}) {
     }
     // A row's keys, their kinds and the words a condition takes are the
     // data-key check's; what it cannot say is a list row's arity, an empty
-    // `from` in one, and a condition the map has no vowels for.
+    // `from` in one, a condition the map has no vowels for, and a letter a
+    // logographic map's row could never meet.
     const hasVowels = typeof vowels === "string" && vowels !== "";
     const rowFindings = (list, keyPath) => {
         if (!Array.isArray(list)) return;
         const label = `data.map.${keyPath.join(".").replace(/\.(\d+)/g, "[$1]")}`;
         list.forEach((row, at) => {
+            const from =
+                Array.isArray(row) ? row[0]
+                : isMap(row) ? row.from
+                : undefined;
+            const letter = typeof from === "string" ? from.match(LETTER)?.[0] : undefined;
+            if (logographic === true && letter !== undefined)
+                findings.push(
+                    finding(
+                        [...keyPath, at],
+                        `\`${label}[${at}]\` matches \`${letter}\`, a letter, and the rows of a ` +
+                            "logographic map write only the text between words",
+                    ),
+                );
             if (Array.isArray(row)) {
                 if (row.length !== 2)
                     findings.push(

package/engine/typographer.mjs

@@ -0,0 +1,248 @@
+/*
+ * This file is part of the Song of Heroic Lands (SoHL) system for Foundry VTT.
+ * Copyright (c) 2024-2026 Tom Rodriguez ("Toasty") — <toasty@heroiclands.org>
+ *
+ * This work is licensed under the GNU General Public License v3.0 (GPLv3).
+ * You may copy, modify, and distribute it under the terms of that license.
+ *
+ * For full terms, see the LICENSE.md file in the project root or visit:
+ * https://www.gnu.org/licenses/gpl-3.0.html
+ *
+ * SPDX-License-Identifier: GPL-3.0-or-later
+ */
+
+/**
+ * **Typographic punctuation on the web.** A web page sets straight quotes as
+ * curly ones and a run of hyphens or full stops as the dash or ellipsis it
+ * stands for, as a printed page does:
+ *
+ * | Written | Set as |
+ * | ------- | ------ |
+ * | `"word"` | “word” |
+ * | `'word'` | ‘word’ |
+ * | `it's`, `'90s`, `'twas` | it’s, ’90s, ’twas |
+ * | `--` | – |
+ * | `---` | — |
+ * | `...` | … |
+ * | `<<`, `>>` | «, » |
+ *
+ * A quote is read by the characters either side of it, as emphasis is: one
+ * that can only open a quotation opens one, one that can only close it closes
+ * a quotation already open earlier on the page, and one that could do either
+ * is left straight. Code, link destinations, raw HTML and a heading's
+ * attribute block are never touched, and neither is a backslash-escaped
+ * character.
+ *
+ * @module
+ */
+
+/** The characters markdown-it's text rule stops at, and the ones this rule reads. */
+const STOPS = new Set([..."\n!#$%&*+-:<=@[\\]^_`{}~", '"', "'", ".", ">"]);
+
+/**
+ * The token a typographic character is written as. It is a token of its own,
+ * not text, so a reader of a heading's text can tell where it stands.
+ */
+export const TYPOGRAPHIC = "typographic";
+
+/** The characters this rule reads. */
+const TRIGGERS = new Set(['"', "'", "-", ".", "<", ">"]);
+
+const LEFT_SINGLE = "‘";
+const RIGHT_SINGLE = "’";
+const LEFT_DOUBLE = "“";
+const RIGHT_DOUBLE = "”";
+
+/** ASCII punctuation, as a byte table reads it. */
+const isAsciiPunct = (char) => char !== undefined && /^[!-/:-@[-`{-~]$/.test(char);
+
+/** ASCII white space, as a byte table reads it. */
+const isAsciiSpace = (char) => char === " " || char === "\t" || char === "\n" || char === "\r";
+
+/** Unicode punctuation. */
+const isPunct = (char) => char !== undefined && /^\p{P}$/u.test(char);
+
+/** Unicode punctuation or a symbol. */
+const isPunctOrSymbol = (char) => char !== undefined && /^[\p{P}\p{S}]$/u.test(char);
+
+/** Unicode white space. */
+const isSpace = (char) => char !== undefined && (isAsciiSpace(char) || /^\s$/u.test(char));
+
+const isLetter = (char) => char !== undefined && /^\p{L}$/u.test(char);
+const isDigit = (char) => char !== undefined && /^\p{Nd}$/u.test(char);
+const isAsciiDigit = (char) => char !== undefined && char >= "0" && char <= "9";
+
+/**
+ * Whether a run of quote characters can open and can close, by the flanking
+ * rules emphasis uses.
+ *
+ * @param {string} before - The character before the run, or a newline.
+ * @param {string} after - The character after it, or a space.
+ * @returns {{canOpen: boolean, canClose: boolean}} What the run can do.
+ */
+function flanking(before, after) {
+    const beforePunct = isPunctOrSymbol(before);
+    const beforeSpace = isSpace(before);
+    const afterPunct = isPunctOrSymbol(after);
+    const afterSpace = isSpace(after);
+    return {
+        canOpen: !afterSpace && (!afterPunct || beforeSpace || beforePunct),
+        canClose: !beforeSpace && (!beforePunct || afterSpace || afterPunct),
+    };
+}
+
+/**
+ * Where a trailing attribute block starts in an inline run, so a heading's
+ * `{#id key="value"}` keeps its straight quotes and hyphens.
+ *
+ * @param {string} src - The inline run.
+ * @returns {number} The index of its opening brace, or the run's length.
+ */
+function attributeStart(src) {
+    const match = /\{[^{}\n]*\}\s*$/.exec(src);
+    return match ? match.index : src.length;
+}
+
+/**
+ * The punctuation one position sets, and how many characters it consumes.
+ *
+ * @param {object} state - markdown-it's inline state.
+ * @returns {{text: string, length: number}|null} The replacement, or `null`
+ *   where the character stays as written.
+ */
+function substitution(state) {
+    const { src, pos } = state;
+    const end = src.indexOf("\n", pos);
+    const line = [...src.slice(pos, end < 0 ? src.length : end + 1)];
+    const c = line[0];
+    if (line.length > 2) {
+        if (c === "-" && line[1] === "-" && line[2] === "-") return { text: "—", length: 3 };
+        if (c === "." && line[1] === "." && line[2] === ".") return { text: "…", length: 3 };
+    }
+    if (c === ".") return null;
+    if (line.length > 1) {
+        if (c === "<") return line[1] === "<" ? { text: "«", length: 2 } : null;
+        if (c === ">") return line[1] === ">" ? { text: "»", length: 2 } : null;
+        if (c === "-" && line[1] === "-") return { text: "–", length: 2 };
+    }
+    if (c !== "'" && c !== '"') return null;
+
+    const before = pos > 0 ? [...src.slice(0, pos)].at(-1) : "\n";
+    let run = 0;
+    while (run < line.length && line[run] === c) run++;
+    const { canOpen, canClose } = flanking(before, run < line.length ? line[run] : " ");
+    const counter = (state.env.typographerQuotes ??= { single: 0, double: 0 });
+    const one = (text) => ({ text, length: 1 });
+
+    if (c === "'") {
+        // A decade: '90s.
+        if (
+            canOpen &&
+            !canClose &&
+            line.length > 3 &&
+            isAsciiDigit(line[1]) &&
+            isAsciiDigit(line[2]) &&
+            line[3] === "s"
+        ) {
+            const after = line.length > 4 ? line[4] : " ";
+            if (isSpace(after) || isPunctOrSymbol(after)) return one(RIGHT_SINGLE);
+        }
+        // An elision: 'twas, 'em, 'net.
+        if (
+            line.length > 1 &&
+            (isPunct(before) || isSpace(before)) &&
+            ["t", "e", "n", "l"].includes(line[1])
+        )
+            return one(RIGHT_SINGLE);
+        // An apostrophe between a letter or digit and a letter.
+        if (line.length > 1 && (isDigit(before) || isLetter(before)) && isLetter(line[1]))
+            return one(RIGHT_SINGLE);
+        if (canOpen && !canClose) {
+            const contracted =
+                (line.length > 1 &&
+                    ["s", "m", "t", "d"].includes(line[1]) &&
+                    (line.length < 3 || isAsciiPunct(line[2]) || isAsciiSpace(line[2]))) ||
+                (line.length > 2 &&
+                    ["ve", "ll", "re"].includes(`${line[1]}${line[2]}`) &&
+                    (line.length < 4 || isAsciiPunct(line[3]) || isAsciiSpace(line[3])));
+            if (contracted) return one(RIGHT_SINGLE);
+            counter.single++;
+            return one(LEFT_SINGLE);
+        }
+        // A plural possessive or a dropped letter: Smiths', doin'.
+        if (line.length > 2 && !isDigit(line[1])) return one(RIGHT_SINGLE);
+        if (counter.single > 0 && closes(canOpen, canClose, line)) {
+            counter.single--;
+            return one(RIGHT_SINGLE);
+        }
+        return null;
+    }
+
+    if (canOpen && !canClose) {
+        counter.double++;
+        return one(LEFT_DOUBLE);
+    }
+    if (counter.double > 0 && closes(canOpen, canClose, line)) {
+        // A measurement in inches: "Monitor 21"".
+        if (line.length > 1 && line[1] === '"' && isDigit(before)) return null;
+        counter.double--;
+        return one(RIGHT_DOUBLE);
+    }
+    return null;
+}
+
+/**
+ * Whether a quote closes: it can only close, or it could do either and stands
+ * before punctuation that ends the line or is followed by white space or more
+ * punctuation.
+ *
+ * @param {boolean} canOpen - The run can open.
+ * @param {boolean} canClose - The run can close.
+ * @param {string[]} line - The rest of the line, from the quote.
+ * @returns {boolean} Whether it closes.
+ */
+function closes(canOpen, canClose, line) {
+    if (canClose && !canOpen) return true;
+    return (
+        canClose &&
+        canOpen &&
+        line.length > 1 &&
+        isPunct(line[1]) &&
+        (line.length === 2 || isAsciiPunct(line[2]) || isAsciiSpace(line[2]))
+    );
+}
+
+/**
+ * Set typographic punctuation in every inline run.
+ *
+ * @param {import("markdown-it")} md - Markdown parser to extend.
+ */
+export function typographerPlugin(md) {
+    md.inline.ruler.at("text", (state, silent) => {
+        let pos = state.pos;
+        while (pos < state.posMax && !STOPS.has(state.src[pos])) pos++;
+        if (pos === state.pos) return false;
+        if (!silent) state.pending += state.src.slice(state.pos, pos);
+        state.pos = pos;
+        return true;
+    });
+    md.inline.ruler.push("typographer", (state, silent) => {
+        if (!TRIGGERS.has(state.src[state.pos])) return false;
+        state.typographerEnd ??= attributeStart(state.src);
+        if (state.pos >= state.typographerEnd) return false;
+        const found = substitution(state);
+        if (!found) return false;
+        if (!silent) state.push(TYPOGRAPHIC, "", 0).content = found.text;
+        state.pos += found.length;
+        return true;
+    });
+    md.renderer.rules[TYPOGRAPHIC] = (tokens, idx) => md.utils.escapeHtml(tokens[idx].content);
+    // An image's description is its text, its punctuation included.
+    const asText = md.renderer.renderInlineAsText.bind(md.renderer);
+    md.renderer.renderInlineAsText = (tokens, options, env) =>
+        tokens
+            .map((token) =>
+                token.type === TYPOGRAPHIC ? token.content : asText([token], options, env),
+            )
+            .join("");
+}