@heroiclands/package-build 25.0.0 → 25.1.0

package/CHANGELOG.md

@@ -1,5 +1,19 @@
 # @heroiclands/package-build
 
+## 25.1.0
+
+**Writing and conversion**
+
+- Native writing now requires an encoding note that chooses its language and font, while unencoded text keeps its Roman spelling.
+- Language runs share Roman punctuation and literal glyph codes, preserving typed spelling and case while locating invalid characters.
+- The `encode` and `decode` commands replace `transliterate`; browser conversion can use vocabulary to recover words and names while preserving stress.
+
+**Rendering and preview**
+
+- Nested emphasis follows the surrounding script, using its red ink or overline convention.
+- Indexed previews show source positions for text they cannot render while keeping the readable preview visible.
+- Ch'alish P'ilam Common text no longer triggers a false character warning.
+
 ## 25.0.0
 
 ### Major Changes

package/README.md

@@ -78,6 +78,7 @@ The documentation lives in the Markdown files in this repository. Start with [th
 | Foundry integration testing                       | [End-to-end testing](docs/guides/e2e.md), including an optional S3 server |
 | Release and deployment                            | [Release and deploy](docs/guides/release-and-deploy.md)                   |
 | Invented languages and scripts                    | [Write in invented scripts](docs/guides/writing-in-invented-scripts.md)   |
+| Encoding rules and the conversion core            | [Encoding core](docs/encoding-core.md)                                    |
 | Buffer-only preview and Gitea rendering           | [Source preview](docs/source-preview.md)                                  |
 | Every CLI command                                 | [Commands](docs/commands.md)                                              |
 | Every build setting                               | [Configuration](docs/configuration.md)                                    |

package/bin/content-commands.mjs

@@ -30,7 +30,6 @@
  *   npx package-build site [--out <dir>]
  *   npx package-build map [--tree] [--from <shortcode>] [--chart <shortcode>] [--travel] [--out <dir>]
  *   npx package-build reachability <dir> [file] [--index <shortcode>]
- *   npx package-build transliterate <map> [--lang <language>] [--font <slug>] [--hex] [text..]
  *   npx package-build lexicon <language> [--map <map>] [--font <slug>] [--derived] [--json]
  *   npx package-build addresses diff --from <zip|dir> [--strict]
  *   npx package-build npc new (--spec <json> | --specfile <path>) [--seed <text>] [--out <path>]
@@ -110,11 +109,10 @@ import { coreReport, coreReportText } from "../engine/core-report.mjs";
 import { checkScriptNoteFonts } from "../engine/script-emphasis.mjs";
 import {
     checkLexiconNatives,
-    checkMapFonts,
+    checkEncodingLexicon,
     languageNoteFindings,
     lexiconEntries,
-    transliterate,
-} from "../engine/map-lexicon.mjs";
+} from "../engine/encoding-lexicon.mjs";
 import { checkFormatting, checkPrettierConventions, lintMarkdown } from "../engine/prose-lint.mjs";
 import { lintProse, scoreProseTree } from "../engine/readability-lint.mjs";
 import { loadPackageBuildConfig } from "../config.mjs";
@@ -321,7 +319,6 @@ export function registerContentCommands(cli) {
         )
         .command(withIndexPreflight(mapCommand()))
         .command(withIndexPreflight(reachabilityCommand()))
-        .command(transliterateCommand())
         .command(lexiconCommand())
         .command(coreCommand())
         .command(addressesCommand())
@@ -1285,15 +1282,22 @@ function lintCommand() {
                 // the languages this package declares, the faces it installed
                 // and the maps its notes declare.
                 const runs = runContext(config, records);
+                const loaderLocations = new Set(
+                    (runs.encodingFindings ?? []).map((one) =>
+                        JSON.stringify([one.file, one.line, one.column, one.message]),
+                    ),
+                );
+                const reportedLoaders = new Set();
                 const findings = [
                     ...index.notes.flatMap((note) =>
                         markupFindings(note.raw, note.file, session.syntax, runs),
                     ),
                     // Each mapping note's faces, each language's native forms
                     // and each language's `note`, against the faces and maps.
-                    ...index.notes.flatMap((note) => checkMapFonts(note, runs)),
                     ...index.notes.flatMap((note) => checkScriptNoteFonts(note, runs)),
                     ...index.notes.flatMap((note) => checkLexiconNatives(note, runs)),
+                    ...index.notes.flatMap((note) => checkEncodingLexicon(note, runs)),
+                    ...(runs.encodingFindings ?? []),
                     ...languageNoteFindings({ ...config, configFile: packConfigPath() }, records),
                     ...index.notes.flatMap(firstUseChecker),
                     ...(names ? [names] : []),
@@ -1309,7 +1313,13 @@ function lintCommand() {
                     ...states.findings,
                     ...generatedArt,
                     ...artSizes,
-                ];
+                ].filter((one) => {
+                    const key = JSON.stringify([one.file, one.line, one.column, one.message]);
+                    if (!loaderLocations.has(key)) return true;
+                    if (reportedLoaders.has(key)) return false;
+                    reportedLoaders.add(key);
+                    return true;
+                });
                 // Only an **error** fails the run. `reportFindings` draws that
                 // line, so the rule is the shared one rather than a second
                 // copy here. Each finding names its own file.
@@ -2566,7 +2576,7 @@ function mapCommand() {
  *   `undefined` when the content tree could not be read whole, which is
  *   reported.
  */
-function mappingContext() {
+function encodingContext() {
     const config = loadPackConfig();
     const problems = [];
     const records = indexRecordsFor({
@@ -2590,7 +2600,7 @@ function mappingContext() {
  * @param {{problem: string, about: string}} found - What is wrong.
  * @param {string} [lang] - The language asked for.
  */
-function reportMapping(config, found) {
+function reportEncoding(config, found) {
     if (found.about === "lang") {
         const file = packConfigPath();
         let text;
@@ -2613,84 +2623,10 @@ function reportMapping(config, found) {
 }
 
 /**
- * `package-build transliterate <map> [text..]` — map a passage exactly as the
- * build maps a run marked with the same `map=`, `lang=` and `font=`.
- *
- * @returns {object} The yargs command module.
- */
-// eslint-disable-next-line
-function transliterateCommand() {
-    return {
-        command: "transliterate <map> [text..]",
-        describe: "Map a passage into a script exactly as the build maps a marked run",
-        builder: (yargs) =>
-            yargs
-                .positional("map", {
-                    type: "string",
-                    describe: "The `data.map.name` of a mapping note",
-                })
-                .positional("text", {
-                    type: "string",
-                    array: true,
-                    describe: "The passage; `-` or nothing reads it from stdin",
-                })
-                .option("lang", {
-                    type: "string",
-                    describe: "A declared language, whose lexicon is read first",
-                })
-                .option("font", {
-                    type: "string",
-                    describe:
-                        "The face; the language's `data.writtenIn` face where the map is drawn in it, and required when neither it nor the map gives one",
-                })
-                .option("hex", {
-                    type: "boolean",
-                    default: false,
-                    describe: "Print the code points, as `U+XXXX` separated by spaces",
-                }),
-        handler: async (argv) => {
-            try {
-                const read = mappingContext();
-                if (!read) {
-                    process.exitCode = 1;
-                    return;
-                }
-                const words = (argv.text ?? []).map(String);
-                const text =
-                    !words.length || (words.length === 1 && words[0] === "-") ?
-                        (await readStdin()).replace(/\r?\n$/, "")
-                    :   words.join(" ");
-                const mapped = transliterate(
-                    read.ctx,
-                    { map: String(argv.map), lang: argv.lang, font: argv.font },
-                    text,
-                );
-                if ("problem" in mapped) {
-                    reportMapping(read.config, mapped);
-                    return;
-                }
-                console.log(
-                    argv.hex ?
-                        Array.from(
-                            mapped.text,
-                            (char) =>
-                                `U+${char.codePointAt(0).toString(16).toUpperCase().padStart(4, "0")}`,
-                        ).join(" ")
-                    :   mapped.text,
-                );
-            } catch (err) {
-                reportFailure(err);
-                process.exitCode = 1;
-            }
-        },
-    };
-}
-
-/**
- * `package-build lexicon <language> [--map <map>]` — every word of a language's
- * lexicon with its native form under a map, as the build writes it; with
+ * `package-build lexicon <language> [--enc <enc>]` — every word of a language's
+ * lexicon with its native form under an encoding, as the build writes it; with
  * `--derived`, the forms its rules derive follow, each naming its word in a
- * sixth column. Under a logographic map the third column says where a
+ * sixth column. Under a logographic enc the third column says where a
  * word's writing comes from, `own` or `parts`, or marks it `none` or
  * `ambiguous`.
  *
@@ -2700,22 +2636,20 @@ function transliterateCommand() {
 function lexiconCommand() {
     return {
         command: "lexicon <language>",
-        describe: "List a language's words with their native forms under a map",
+        describe: "List a language's words with their native forms under an encoding",
         builder: (yargs) =>
             yargs
                 .positional("language", {
                     type: "string",
                     describe: "A declared language whose `note` names its language note",
                 })
-                .option("map", {
+                .option("enc", {
                     type: "string",
-                    describe:
-                        "The `data.map.name` of a mapping note; the language's `data.writtenIn` map where left out",
+                    describe: "The encoding document Address, with doc as default type",
                 })
                 .option("font", {
                     type: "string",
-                    describe:
-                        "The face; the language's `data.writtenIn` face where the map is drawn in it, and required when neither it nor the map gives one",
+                    describe: "The encoding document's font; an explicit value must agree",
                 })
                 .option("derived", {
                     type: "boolean",
@@ -2730,19 +2664,19 @@ function lexiconCommand() {
                 }),
         handler: (argv) => {
             try {
-                const read = mappingContext();
+                const read = encodingContext();
                 if (!read) {
                     process.exitCode = 1;
                     return;
                 }
                 const listed = lexiconEntries(read.ctx, {
                     lang: String(argv.language),
-                    map: argv.map === undefined ? undefined : String(argv.map),
+                    enc: argv.enc === undefined ? undefined : String(argv.enc),
                     font: argv.font,
                     derived: Boolean(argv.derived),
                 });
                 if ("problem" in listed) {
-                    reportMapping(read.config, listed);
+                    reportEncoding(read.config, listed);
                     return;
                 }
                 if (argv.json) console.log(JSON.stringify(listed.entries, null, 2));
@@ -3339,7 +3273,7 @@ function readNpcSpec(argv) {
  *
  * @param {{file?: string, raw?: string}} note - Where the note is.
  * @param {Array<string|number>} keyPath - The frontmatter key at fault, or the
- *   map a missing key belongs in.
+ *   enc a missing key belongs in.
  * @param {string} message - What is wrong.
  * @returns {Error} The error, for `reportFailure` to print as a diagnostic.
  */

package/bin/encoding-commands.mjs

@@ -0,0 +1,146 @@
+/*
+ * This file is part of the Song of Heroic Lands (SoHL) system for Foundry VTT.
+ * Copyright (c) 2024-2026 Tom Rodriguez ("Toasty") — <toasty@heroiclands.org>
+ *
+ * This work is licensed under the GNU General Public License v3.0 (GPLv3).
+ * You may copy, modify, and distribute it under the terms of that license.
+ *
+ * For full terms, see the LICENSE.md file in the project root or visit:
+ * https://www.gnu.org/licenses/gpl-3.0.html
+ *
+ * SPDX-License-Identifier: GPL-3.0-or-later
+ */
+/** Bare encoding commands: a published definition, UTF-8 input and exact output. */
+import fs from "node:fs";
+import { encodeText, decodeText } from "../engine/encoding.mjs";
+import { encodingAddress } from "../engine/encoding-notes.mjs";
+import { readStdin } from "../engine/read-stdin.mjs";
+
+/** Apply one of the bare commands; all conversion is shared with the build. */
+async function execute(direction, argv) {
+    const strings = argv.text ?? [];
+    if (
+        !argv.enc ||
+        (argv.in !== undefined && strings.length) ||
+        (argv.in === undefined && strings.length !== 1)
+    ) {
+        console.error(
+            `package-build ${direction}: --enc is required; give one string or --in <file|->`,
+        );
+        process.exitCode = 2;
+        return;
+    }
+    let source = "<text>";
+    try {
+        let encoding;
+        if (argv.enc !== "default") {
+            const [{ loadPackConfig }, { indexRecordsFor }, { runContext }] = await Promise.all([
+                import("../engine/pack-config.mjs"),
+                import("../engine/content-index.mjs"),
+                import("../engine/content-scripts.mjs"),
+            ]);
+            const config = loadPackConfig(),
+                problems = [];
+            const records = indexRecordsFor({ config, problems });
+            if (problems.length) throw new Error(problems.map((one) => one.message).join("\n"));
+            const context = runContext(config, records);
+            if (context.encodingFindings?.length)
+                throw Object.assign(
+                    new Error(context.encodingFindings[0].message),
+                    context.encodingFindings[0],
+                );
+            encoding =
+                context.encodings.get(String(argv.enc)) ??
+                context.encodings.get(encodingAddress(String(argv.enc), config.contentPackage));
+            if (!encoding) throw new Error(`Encoding document \`${argv.enc}\` does not resolve`);
+        }
+        source =
+            argv.in === "-" ? "<stdin>"
+            : argv.in === undefined ? "<text>"
+            : String(argv.in);
+        const input =
+            argv.in === undefined ? String(strings[0])
+            : argv.in === "-" ? await readStdin()
+            : fs.readFileSync(String(argv.in), "utf8");
+        let output;
+        try {
+            output =
+                !encoding ? input
+                : direction === "encode" ? encodeText(encoding, input)
+                : decodeText(encoding, input);
+        } catch (error) {
+            error.file = source;
+            throw error;
+        }
+        if (argv.out !== undefined) fs.writeFileSync(String(argv.out), output, "utf8");
+        else process.stdout.write(output + (argv.in === undefined ? "\n" : ""));
+    } catch (error) {
+        const file = error.file ?? source;
+        const position = error.line !== undefined ? `:${error.line}:${error.column ?? 1}` : "";
+        console.error(`${file}${position}: error: ${error.message}`);
+        process.exitCode = 1;
+    }
+}
+
+/** Encode roman text through one explicitly selected encoding. */
+function encodeCommand() {
+    return {
+        command: "encode [text..]",
+        describe: "Encode roman text using an encoding document",
+        builder: (yargs) =>
+            yargs
+                .positional("text", { type: "string", array: true, describe: "One input string" })
+                .option("enc", {
+                    type: "string",
+                    requiresArg: true,
+                    describe: "Required encoding document Address, or default",
+                })
+                .option("in", {
+                    alias: "i",
+                    type: "string",
+                    requiresArg: true,
+                    describe: "Read UTF-8 input from a file or - for stdin",
+                })
+                .option("out", {
+                    alias: "o",
+                    type: "string",
+                    requiresArg: true,
+                    describe: "Write the exact result to a file",
+                }),
+        handler: (argv) => execute("encode", argv),
+    };
+}
+
+/** Decode code points through one explicitly selected encoding. */
+function decodeCommand() {
+    return {
+        command: "decode [text..]",
+        describe: "Decode code points using an encoding document",
+        builder: (yargs) =>
+            yargs
+                .positional("text", { type: "string", array: true, describe: "One input string" })
+                .option("enc", {
+                    type: "string",
+                    requiresArg: true,
+                    describe: "Required encoding document Address, or default",
+                })
+                .option("in", {
+                    alias: "i",
+                    type: "string",
+                    requiresArg: true,
+                    describe: "Read UTF-8 input from a file or - for stdin",
+                })
+                .option("out", {
+                    alias: "o",
+                    type: "string",
+                    requiresArg: true,
+                    describe: "Write the exact result to a file",
+                }),
+        handler: (argv) => execute("decode", argv),
+    };
+}
+
+/** Register the two bare conversion commands on the shared CLI. */
+export function registerEncodingCommands(yargs) {
+    return yargs.command(encodeCommand()).command(decodeCommand());
+}

package/bin/package-build.mjs

@@ -145,6 +145,7 @@ import {
     INIT_KINDS,
     INIT_LICENSES,
 } from "../engine/project-init.mjs";
+import { registerEncodingCommands } from "./encoding-commands.mjs";
 import { registerContentCommands } from "./content-commands.mjs";
 
 /**
@@ -711,6 +712,10 @@ function manifestCommand() {
                 // So a `packFolders` finding names the line it is about.
                 configFile: packConfigPath(),
             });
+            if (config.converterData) {
+                const { writeConverterData } = await import("../engine/converter-data.mjs");
+                writeConverterData({ config: shared, packageBuild: config });
+            }
             console.log(
                 `✅ Wrote ${path.relative(config.rootDir, written)} ` +
                     `(${Object.keys(manifest).length} keys, ` +
@@ -1746,30 +1751,32 @@ function bumpCommand() {
 }
 
 registerContentCommands(
-    yargs(hideBin(process.argv))
-        .scriptName("package-build")
-        .command(initCommand())
-        .command(ciCommand())
-        .command(cleanCommand())
-        .command(assetsCommand())
-        .command(stageCommand())
-        .command(calendarsCommand())
-        .command(datefromCommand())
-        .command(datetoCommand())
-        .command(manifestCommand())
-        .command(siteRootCommand())
-        .command(typesCommand())
-        .command(schemaCommand())
-        .command(langCommand())
-        .command(bumpCommand())
-        .command(yamlCommand())
-        .command(changelogCommand())
-        .command(bundleCommand())
-        .command(releaseCommand())
-        .command(deployCommand())
-        .command(cdnCommand())
-        .command(containerCommand())
-        .command(e2eCommand()),
+    registerEncodingCommands(
+        yargs(hideBin(process.argv))
+            .scriptName("package-build")
+            .command(initCommand())
+            .command(ciCommand())
+            .command(cleanCommand())
+            .command(assetsCommand())
+            .command(stageCommand())
+            .command(calendarsCommand())
+            .command(datefromCommand())
+            .command(datetoCommand())
+            .command(manifestCommand())
+            .command(siteRootCommand())
+            .command(typesCommand())
+            .command(schemaCommand())
+            .command(langCommand())
+            .command(bumpCommand())
+            .command(yamlCommand())
+            .command(changelogCommand())
+            .command(bundleCommand())
+            .command(releaseCommand())
+            .command(deployCommand())
+            .command(cdnCommand())
+            .command(containerCommand())
+            .command(e2eCommand()),
+    ),
 )
     .demandCommand(1, "Name a command.")
     .strict()
@@ -1779,5 +1786,9 @@ registerContentCommands(
     .epilog("Run package-build <command> --help for that command's actions and options.")
     .fail((message, err, cli) => {
         if (!err) cli.showHelp("error");
+        if (["encode", "decode"].includes(process.argv[2])) {
+            console.error(`package-build: ${err?.message ?? message}`);
+            process.exit(2);
+        }
         die(err ?? message);
     }).argv;

package/config.mjs

@@ -70,6 +70,7 @@ const SECTION_KEYS = [
     "baseStyles",
     "manifest",
     "manifestFlags",
+    "converterData",
     "schema",
     "clean",
     "lang",
@@ -338,6 +339,28 @@ function normalizeManifest(value) {
     return Object.freeze(structuredClone(input));
 }
 
+/** The optional converter artifact is a JSON file beneath the package stage. */
+function normalizeConverterData(value) {
+    if (value === undefined) return null;
+    const declared = requireNonEmptyString(value, "packageBuild.converterData");
+    const pieces = declared.split(/[\\/]/);
+    if (
+        path.posix.isAbsolute(declared) ||
+        path.win32.isAbsolute(declared) ||
+        pieces.includes("..") ||
+        pieces.includes("") ||
+        !declared.endsWith(".json") ||
+        declared.includes("\0") ||
+        ["module.json", "system.json", "package.json"].includes(declared)
+    ) {
+        fail(
+            "packageBuild.converterData",
+            "must name a JSON artifact within the build stage, such as data/converter.json",
+        );
+    }
+    return pieces.join("/");
+}
+
 /**
  * Normalize a setting that is one glob or several.
  *
@@ -565,6 +588,7 @@ function normalizeExceptions(value, field, where) {
  *                                   named first in the manifest's `styles`.
  * @property {Readonly<Record<string, unknown>>} manifest  The manifest
  *                                   specification, emitted as declared.
+ * @property {string|null} converterData  Optional JSON path relative to the build stage.
  * @property {string|null} manifestFlags  Module to load a `flags` function
  *                                   from, for namespaced flags a repository has
  *                                   to compute. `null` when it declares none.
@@ -938,6 +962,7 @@ export function resolvePackageBuildConfig(shared) {
                     shared.rootDir,
                 ),
         manifest: normalizeManifest(section.manifest),
+        converterData: normalizeConverterData(section.converterData),
         manifestFlags:
             section.manifestFlags === undefined ?
                 null