@heroiclands/package-build 24.1.1 → 25.1.0

package/convert.mjs

@@ -0,0 +1,182 @@
+/*
+ * This file is part of the Song of Heroic Lands (SoHL) system for Foundry VTT.
+ * Copyright (c) 2024-2026 Tom Rodriguez ("Toasty") — <toasty@heroiclands.org>
+ *
+ * This work is licensed under the GNU General Public License v3.0 (GPLv3).
+ * You may copy, modify, and distribute it under the terms of that license.
+ *
+ * For full terms, see the LICENSE.md file in the project root or visit:
+ * https://www.gnu.org/licenses/gpl-3.0.html
+ *
+ * SPDX-License-Identifier: GPL-3.0-or-later
+ */
+/** Browser conversion from one module's versioned, self-contained data file. */
+import { renderAddress } from "./engine/address-render.mjs";
+import { compileEncoding, encodeText, decodeText, EncodingError } from "./engine/encoding.mjs";
+import { encodeWithLanguage, decodeWithLanguage } from "./engine/encoding-language.mjs";
+/**
+ * @typedef {{language:string,data:Record<string,any>}} ConverterLexicon
+ * @typedef {{name:string,lexicon?:ConverterLexicon}} ConverterLanguage
+ * @typedef {{format:2,contentPackage:string,encodings:Record<string,any>,languages:Record<string,ConverterLanguage>}} ConverterData
+ * @typedef {{enc?:string,font?:string,verse?:boolean}} ConversionOptions
+ */
+const compiled = new WeakMap();
+const validated = new WeakSet();
+const object = (value) => value !== null && typeof value === "object" && !Array.isArray(value);
+function keys(value, allowed, where) {
+    if (!object(value)) throw new TypeError(`${where} must be an object`);
+    for (const key of Object.keys(value))
+        if (!allowed.includes(key)) throw new TypeError(`${where}.${key} is unknown`);
+}
+function definition(value, name) {
+    if (!object(value)) throw new TypeError("An encoding definition must be an object");
+    keys(value, ["data", "codes"], `encodings.${name}`);
+    let known = compiled.get(value);
+    if (!known) {
+        known = compileEncoding(value.data, { codes: value.codes, name });
+        compiled.set(value, known);
+    }
+    return known;
+}
+
+function lexicalData(value, where) {
+    keys(value, ["words", "affixes", "grammar", "register"], where);
+    for (const field of ["words", "affixes", "register"]) {
+        if (value[field] === undefined) continue;
+        if (!Array.isArray(value[field])) throw new TypeError(`${where}.${field} must be an array`);
+        for (const row of value[field]) {
+            const name = field === "register" ? "name" : "form";
+            if (!object(row) || typeof row[name] !== "string" || !row[name])
+                throw new TypeError(`${where}.${field} needs ${name}`);
+            if (
+                row.native !== undefined &&
+                (!Array.isArray(row.native) ||
+                    row.native.some(
+                        (native) =>
+                            !object(native) ||
+                            typeof native.enc !== "string" ||
+                            typeof native.text !== "string" ||
+                            (native.font !== undefined && typeof native.font !== "string"),
+                    ))
+            )
+                throw new TypeError(`${where}.${field} has invalid native writings`);
+        }
+    }
+    if (value.grammar !== undefined && !object(value.grammar))
+        throw new TypeError(`${where}.grammar must be an object`);
+}
+
+/** Validate the format-2 converter data without opening files or using Node.
+ * @param {ConverterData} data Parsed JSON, treated as read-only.
+ * @returns {ConverterData} The validated data.
+ */
+export function validateData(data) {
+    if (validated.has(data)) return data;
+    keys(data, ["format", "contentPackage", "encodings", "languages"], "data");
+    if (data.format !== 2) throw new TypeError("Converter data format must be 2");
+    if (
+        typeof data.contentPackage !== "string" ||
+        !/^[a-z0-9]+(?:-[a-z0-9]+)*$/u.test(data.contentPackage)
+    )
+        throw new TypeError("Converter data needs its canonical contentPackage");
+    if (!object(data.encodings) || !object(data.languages))
+        throw new TypeError("Converter encodings and languages must be objects");
+    for (const [name, value] of Object.entries(data.encodings)) {
+        if (!/^[a-z0-9]+(?:-[a-z0-9]+)*-note-doc-[a-z0-9]+$/u.test(name))
+            throw new TypeError(`Invalid encoding document Address ${name}`);
+        const encoding = definition(value, name);
+        if (!Object.hasOwn(data.languages, encoding.language))
+            throw new TypeError(`Encoding ${name} needs configured language ${encoding.language}`);
+    }
+    for (const [language, value] of Object.entries(data.languages)) {
+        keys(value, ["name", "lexicon"], `languages.${language}`);
+        if (typeof value.name !== "string")
+            throw new TypeError(`Language ${language} needs its name`);
+        if (value.lexicon !== undefined) {
+            keys(value.lexicon, ["language", "data"], `languages.${language}.lexicon`);
+            if (typeof value.lexicon.language !== "string" || !object(value.lexicon.data))
+                throw new TypeError(`Language ${language} has invalid lexicon inputs`);
+            lexicalData(value.lexicon.data, `languages.${language}.lexicon.data`);
+        }
+    }
+    validated.add(data);
+    return data;
+}
+
+function conversion(data, language, text, options, direction) {
+    validateData(data);
+    if (typeof text !== "string") throw new TypeError("Conversion input must be text");
+    let entry = language === null || language === undefined ? undefined : data.languages[language];
+    if (language !== null && language !== undefined && !Object.hasOwn(data.languages, language))
+        throw new TypeError(`Unknown language ${language}`);
+    const name = options.enc ?? "default";
+    if (name === "default") return { text, findings: [] };
+    const short = /^(?:(?:note-)?doc-|doc\/)?([a-z0-9]+)$/u.exec(name);
+    const selected =
+        Object.hasOwn(data.encodings, name) ? name
+        : short ?
+            renderAddress({
+                package: data.contentPackage,
+                system: "note",
+                type: "doc",
+                shortcode: short[1],
+            })
+        :   undefined;
+    if (!selected || !Object.hasOwn(data.encodings, selected))
+        throw new TypeError(`Unknown encoding document ${name}`);
+    const encoding = definition(data.encodings[selected], selected);
+    if (entry && language !== encoding.language)
+        throw new TypeError(`Encoding ${selected} writes ${encoding.language}, not ${language}`);
+    entry ??= data.languages[encoding.language];
+    const settings = {
+        font: options.font ?? encoding.font,
+        verse: options.verse,
+        lexicon: entry?.lexicon,
+    };
+    if (entry?.lexicon)
+        return direction === "encode" ?
+                encodeWithLanguage(encoding, text, settings)
+            :   decodeWithLanguage(encoding, text, settings);
+    try {
+        return {
+            text:
+                direction === "encode" ?
+                    encodeText(encoding, text, settings)
+                :   decodeText(encoding, text, settings),
+            findings: [],
+        };
+    } catch (error) {
+        if (!(error instanceof EncodingError)) throw error;
+        return {
+            text: "",
+            findings: [
+                {
+                    severity: "error",
+                    message: error.message,
+                    offset: error.offset ?? 0,
+                    line: error.line ?? 1,
+                    column: error.column ?? 1,
+                },
+            ],
+        };
+    }
+}
+
+/** Encode with the selected encoding document's language, optionally stated explicitly.
+ * @param {ConverterData} data The module's parsed converter data.
+ * @param {string|null|undefined} language A language key, or null to infer it from the encoding.
+ * @param {string} text Roman input.
+ * @param {ConversionOptions} [options] Encoding, face or verse overrides.
+ */
+export function encode(data, language, text, options = {}) {
+    return conversion(data, language, text, options, "encode");
+}
+/** Decode with the selected encoding document's optional language lexicon.
+ * @param {ConverterData} data The module's parsed converter data.
+ * @param {string|null|undefined} language A language key, or null to infer it from the encoding.
+ * @param {string} text Encoded Unicode input.
+ * @param {ConversionOptions} [options] Encoding or face overrides.
+ */
+export function decode(data, language, text, options = {}) {
+    return conversion(data, language, text, options, "decode");
+}

package/docs/api.md

@@ -1343,6 +1343,13 @@ index for checking page links.
 
 `renderPreview(text, { dark = false, infobox = true, file = "", currentDate = "720.1", index })` renders the current Markdown buffer directly to HTML. Frontmatter is optional. Without `index` it needs no package configuration, saved note, content tree, dependency cache, database or Pandoc installation. The result is `{ ok, html, frontmatter, findings }`; `file` only labels diagnostics. Malformed frontmatter produces a warning and remains visible without preventing a preview update.
 
+With `index`, every marked-text run error appears in `findings` as
+`{ file, severity: "error", line, column, message }`. Positions refer to the
+authored buffer, including frontmatter lines, before indexed expansion. Any
+error finding makes `ok` false. `html` is returned even when `ok` is false, with
+recoverable Roman text for a native-script run that cannot be set. Without
+`index`, marked text remains source and produces no run findings.
+
 With `index`, from [`engine.previewIndex`](#enginepreviewindex), the preview expands the buffer as the website expands a page and renders it through the same renderer: links resolve, embeds draw their pictures, marked text is mapped and set in its face, icons draw, figure numbers carry their asset roles, and a concealed link is concealed. A query, which a buffer cannot run, stays highlighted source.
 
 | Export                   | Signature                       | Returns                               | Use it when                                                            |
@@ -1384,17 +1391,30 @@ Linting a content tree's **addresses** — the rules every package's notes are a
 
 The charset authored content is held to, so a book can choose its face. The packs and the website render in whatever font the reader's browser or Foundry supplies, and a glyph nobody has is somebody else's problem. **A book is not that.** A PDF embeds the faces it sets, so every character in the corpus is a claim on the book's typeface — and the claim is silent, which is what makes it expensive.
 
-| Export               | Signature                                  | Returns                                                                                                                                                                  | Use it when                                                                |
-| -------------------- | ------------------------------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | -------------------------------------------------------------------------- |
-| `isLetterTier`       | `function isLetterTier(cp)`                | {boolean} Whether Tier 1 admits it.                                                                                                                                      | Tier 1 — the letters, and the two whitespace characters a file is made of. |
-| `TYPOGRAPHY`         | `const TYPOGRAPHY`                         | —                                                                                                                                                                        | Tier 2 — typography, enumerated one codepoint at a time.                   |
-| `NOTATION`           | `const NOTATION`                           | —                                                                                                                                                                        | Tier 3 — the notation the rules and price tables are written in.           |
-| `isAllowedCodePoint` | `function isAllowedCodePoint(cp)`          | {boolean} Whether it is allowed outside a code fence.                                                                                                                    | Whether the charset admits a code point anywhere in a note.                |
-| `isDiagramCodePoint` | `function isDiagramCodePoint(cp)`          | {boolean} Whether a fence may carry it.                                                                                                                                  | Whether a code point is diagram furniture, admitted inside a fence only.   |
-| `refusalFor`         | `function refusalFor(cp)`                  | {string} A clause naming what it is and what to do instead.                                                                                                              | The reason a code point is refused.                                        |
-| `decomposedRuns`     | `function decomposedRuns(text)`            | {Array<{sequence: string, composed: string, index: number}>} Each offending run, in the order it appears.                                                                | Every non-NFC run in a string, with the composed form it should have been. |
-| `checkText`          | `function checkText(text, file)`           | {Array<{file: string, line: number, column: number, severity: "warning", message: string}>} What is wrong, in file order.                                                | Check one file's text against the charset and the normalization rule.      |
-| `lintContentCharset` | `function lintContentCharset(contentBase,` | `{{findings: Array<{file: string, line: number, column: number, severity: "warning", message: string}>, files: number}}` The findings, and how many files produced them. | Walk a content tree and check every authored file in it.                   |
+| Export                     | Signature                                  | Returns                                                                                                                                                                  | Use it when                                                                                  |
+| -------------------------- | ------------------------------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | -------------------------------------------------------------------------------------------- |
+| `isLetterTier`             | `function isLetterTier(cp)`                | {boolean} Whether Tier 1 admits it.                                                                                                                                      | Tier 1 — the letters, and the two whitespace characters a file is made of.                   |
+| `TYPOGRAPHY`               | `const TYPOGRAPHY`                         | —                                                                                                                                                                        | Tier 2 — typography, enumerated one codepoint at a time.                                     |
+| `NOTATION`                 | `const NOTATION`                           | —                                                                                                                                                                        | Tier 3 — the notation the rules and price tables are written in.                             |
+| `isSettingScriptCodePoint` | `isSettingScriptCodePoint(cp)`             | `boolean`                                                                                                                                                                | recognizing native setting-script input before the selected face checks the individual glyph |
+| `isAllowedCodePoint`       | `function isAllowedCodePoint(cp)`          | {boolean} Whether it is allowed outside a code fence.                                                                                                                    | Whether the charset admits a code point anywhere in a note.                                  |
+| `isDiagramCodePoint`       | `function isDiagramCodePoint(cp)`          | {boolean} Whether a fence may carry it.                                                                                                                                  | Whether a code point is diagram furniture, admitted inside a fence only.                     |
+| `refusalFor`               | `function refusalFor(cp)`                  | {string} A clause naming what it is and what to do instead.                                                                                                              | The reason a code point is refused.                                                          |
+| `decomposedRuns`           | `function decomposedRuns(text)`            | {Array<{sequence: string, composed: string, index: number}>} Each offending run, in the order it appears.                                                                | Every non-NFC run in a string, with the composed form it should have been.                   |
+| `checkText`                | `function checkText(text, file)`           | {Array<{file: string, line: number, column: number, severity: "warning", message: string}>} What is wrong, in file order.                                                | Check one file's text against the charset and the normalization rule.                        |
+| `lintContentCharset`       | `function lintContentCharset(contentBase,` | `{{findings: Array<{file: string, line: number, column: number, severity: "warning", message: string}>, files: number}}` The findings, and how many files produced them. | Walk a content tree and check every authored file in it.                                     |
+
+### `engine.romanCharacters`
+
+Exact permitted Roman source characters for configured languages, before script
+encoding. Inventories come from `data.roman` on each language's own note.
+
+| Export                   | Signature                                | Returns                       | Use it when                                                                                   |
+| ------------------------ | ---------------------------------------- | ----------------------------- | --------------------------------------------------------------------------------------------- |
+| `isRomanWhitespace`      | `isRomanWhitespace(char)`                | `boolean`                     | recognizing standard spaces and ASCII tab or line whitespace                                  |
+| `romanInventoryProblems` | `romanInventoryProblems(value)`          | `string[]`                    | checking NFC, repeated characters and forbidden whitespace or controls in a literal inventory |
+| `romanInventoriesOf`     | `romanInventoriesOf(records, languages)` | `Map<string, RomanInventory>` | looking up a configured language's own inventory by name or BCP 47 tag                        |
+| `romanTextProblems`      | `romanTextProblems(text, inventory)`     | `{offset, message}[]`         | validating standard Roman punctuation and language additions with exact UTF-16 offsets        |
 
 ### `engine.pathCharset`
 
@@ -2761,3 +2781,126 @@ language contributes words, affixes and elements, in that order. Words and
 affixes supply `gloss`, including an affix matched by its `front` form. Elements
 supply `sense` and match their `form` and `variants`. Exact matches precede matches without joining hyphens; case and
 accents are preserved. A missing gloss returns `null`.
+
+### `engine.encoding`
+
+Browser-safe compilation of an encoding document with separately resolved font glyph codes;
+see [encoding text](encoding-core.md). No files are read. Text errors expose a
+UTF-16 `offset` and one-based code-point `line` and `column`; definition errors
+expose their schema `key`.
+
+| Export            | Signature                                   | Returns              | Use it when                                                                         |
+| ----------------- | ------------------------------------------- | -------------------- | ----------------------------------------------------------------------------------- |
+| `EncodingError`   | `new EncodingError(message, details)`       | error                | locating a schema fault or unidentified character                                   |
+| `compileEncoding` | `compileEncoding(data, {codes, name})`      | compiled encoding    | validating executable document mappings and rules                                   |
+| `compileRows`     | `compileRows(definition)`                   | internal row program | executing captured row-conformance fixtures; application inputs use compileEncoding |
+| `validateSource`  | `validateSource(encoding, text, options)`   | void                 | validating accepted Roman spelling and rules independently of mapping coverage      |
+| `encodePieces`    | `encodePieces(encoding, text, options)`     | piece array          | retaining original UTF-16 `start` and `end` offsets while encoding                  |
+| `encodeText`      | `encodeText(encoding, text, {font, verse})` | string               | encoding exactly by rows and inline codes                                           |
+| `decodeText`      | `decodeText(encoding, text, {font})`        | string               | decoding by longest output matches                                                  |
+
+### `engine.encodingLanguage`
+
+Pure language-aware conversion takes a compiled encoding and optional plain
+`lexicon: {language, data}` from `serializeLexicon`. The build and browser use
+the same functions. Findings retain input offsets; an encoding refusal returns
+no partial encoded output. Decoding preserves the plain skeleton of an unknown
+unmarked word and reports it.
+
+| Export               | Signature                                                    | Returns                    | Use it when                                                            |
+| -------------------- | ------------------------------------------------------------ | -------------------------- | ---------------------------------------------------------------------- |
+| `encodeWithLanguage` | `encodeWithLanguage(encoding, text, {font, verse, lexicon})` | `{text, pieces, findings}` | writing authored native entries or complete parts before encoding rows |
+| `decodeWithLanguage` | `decodeWithLanguage(encoding, text, {font, lexicon})`        | `{text, findings}`         | reading native exceptions and recovering unmarked lexical skeletons    |
+
+### `engine.encodingNotes`
+
+Node-side document resolution reads encoding records from the content index
+and glyph codes from the selected font's `fonts.json`. It opens no font encoding
+files and enforces no encoding-file release floor. Diagnostics retain the
+note or font manifest's own source position.
+
+| Export                 | Signature                                      | Returns                              | Use it when                                                   |
+| ---------------------- | ---------------------------------------------- | ------------------------------------ | ------------------------------------------------------------- |
+| `encodingAddress`      | `encodingAddress(value, contentPackage)`       | string or undefined                  | resolving a document Address with default type doc            |
+| `encodingData`         | `encodingData(data)`                           | plain executable data                | separating ordinary document metadata                         |
+| `fontCodes`            | `fontCodes(fonts, font)`                       | code-to-Unicode object               | reading only the selected font's published glyph codes        |
+| `encodingsFromRecords` | `encodingsFromRecords(records, config, fonts)` | `{encodings, definitions, findings}` | compiling addressed documents and preparing serialized inputs |
+
+### `engine.encodingLexicon`
+
+Node-side note projection and validation. `serializeLexicon` emits only word,
+affix, grammar and name-register data read by conversion; the result contains
+no Maps or callbacks.
+
+| Export                  | Signature                                             | Returns                          | Use it when                                                                          |
+| ----------------------- | ----------------------------------------------------- | -------------------------------- | ------------------------------------------------------------------------------------ |
+| `lexiconsFromRecords`   | `lexiconsFromRecords(records, config)`                | Map                              | projecting language words, including nearest complete inherited definitions          |
+| `languageRecordsByName` | `languageRecordsByName(records, config)`              | Map                              | finding each configured language note                                                |
+| `languageNoteFindings`  | `languageNoteFindings(config, records)`               | findings                         | locating configured language notes or Roman inventories that are missing             |
+| `serializeLexicon`      | `serializeLexicon(lexicon)`                           | plain JSON object                | supplying converter input to a browser or another process                            |
+| `encodingFor`           | `encodingFor(context, {enc, font, lang})`             | selected conversion or undefined | choosing an addressed document, its face and optional language lexicon               |
+| `fontProblem`           | `fontProblem(encoding, font)`                         | string or undefined              | explaining an unsupported face choice                                                |
+| `lexiconEntries`        | `lexiconEntries(context, {lang, enc, font, derived})` | entries or problem               | listing authored or derived forms with own, parts or row spellings                   |
+| `checkLexiconNatives`   | `checkLexiconNatives(note, context)`                  | findings                         | validating encoding, face, duplicate slots and visible native code points            |
+| `checkEncodingLexicon`  | `checkEncodingLexicon(note, context)`                 | findings                         | checking every authoritative word and name against its language's encoding documents |
+| `NATIVE_COLUMNS`        | constant                                              | column names                     | reading the natives SQL projection, including its enc column                         |
+| `nativeRows`            | `nativeRows(records)`                                 | rows                             | projecting authored native entries in source order                                   |
+
+## `./convert`
+
+### Browser conversion
+
+`@heroiclands/package-build/convert` exports:
+
+| Export         | Signature                                | Returns                                                    |
+| -------------- | ---------------------------------------- | ---------------------------------------------------------- |
+| `encode`       | `encode(data, language, text, options?)` | Converted text, findings and source pieces for a language. |
+| `decode`       | `decode(data, language, text, options?)` | Roman text and findings.                                   |
+| `validateData` | `validateData(data)`                     | The validated format-2 data.                               |
+
+Its module graph uses no Node built-in and can be bundled for
+a browser without shims. Load the module's [format-2 data file](reference/format-details.md#converter-data-format)
+once and treat it as read-only.
+
+```js
+import { encode, decode, validateData } from "@heroiclands/package-build/convert";
+const data = validateData(await (await fetch("data/converter.json")).json());
+const native = encode(data, "aelw", "say", { enc: "aelwsemaronenc" });
+const roman = decode(data, "aelw", native.text, { enc: "aelwsemaronenc" });
+const inferred = encode(data, null, "say", { enc: "aelwsemaronenc" });
+```
+
+Both functions take `(data, language, text, options?)` and return
+`{text, findings}`; encoding with a language also returns source-positioned
+`pieces`. A language key selects its encoding, face and lexical inputs.
+`options.enc` and `options.font` override the encoding and face; `options.verse`
+asks the encoder to interpret verse structure. With `language: null`, `enc` is
+required and only the encoding's rows apply.
+
+Language conversion uses the same pure `encodeWithLanguage` and
+`decodeWithLanguage` functions as the build. Recorded native writings and
+parts precede encoding rows, and decoding can recover an unmarked word from
+the lexicon. An unknown word gives a positioned finding. A bare refusal
+returns empty text and an error finding. Findings carry `severity`, `message`,
+`offset` (UTF-16), and one-based Unicode-code-point `line` and `column`.
+Invalid data, unknown languages and missing encoding selections throw an
+error before conversion. `validateData(data)` returns the validated data or
+throws; it opens no files.
+
+The Node-side `engine/converter-data.mjs` module prepares these inputs from an
+indexed package:
+
+| Export               | Signature                                              | Returns                                                                       |
+| -------------------- | ------------------------------------------------------ | ----------------------------------------------------------------------------- |
+| `buildConverterData` | `buildConverterData(config, records)`                  | Format-1 converter data from the build context.                               |
+| `writeConverterData` | `writeConverterData({config, packageBuild, records?})` | `{path, data}` when the optional output is configured; otherwise `undefined`. |
+
+The content run scanner reports diagnostic columns in Unicode code points.
+Its run offsets, run columns and SQL `runs.column` values retain UTF-16 source
+units. Encoding-file diagnostics retain their own file and position.
+
+An omitted or null language is inferred from the selected encoding document,
+including its lexical native overrides and decoding recovery. Short document
+references resolve in `data.contentPackage`; foreign encodings require their
+full canonical Address. Use the pure engine's `encodeText` and `decodeText`
+when conversion must ignore vocabulary, as the CLI commands do.