@heroiclands/package-build 25.1.0 → 25.1.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +19 -0
- package/bin/content-commands.mjs +43 -6
- package/docs/api.md +7 -0
- package/docs/encoding-core.md +65 -7
- package/docs/guides/writing-in-invented-scripts.md +7 -0
- package/docs/reference/format-details.md +66 -63
- package/docs/reference/note-types.md +54 -51
- package/engine/content-alerts.mjs +2 -2
- package/engine/content-blocks.mjs +2 -2
- package/engine/content-index.mjs +23 -1
- package/engine/content-scripts.mjs +15 -6
- package/engine/content-scroll.mjs +2 -4
- package/engine/content-session.mjs +1 -0
- package/engine/content-timings.mjs +13 -1
- package/engine/content-verse.mjs +2 -2
- package/engine/dictionary-rows.mjs +14 -2
- package/engine/encoding-language.mjs +47 -20
- package/engine/encoding-lexicon.mjs +17 -3
- package/engine/encoding-template.mjs +99 -42
- package/engine/encoding-terms.mjs +42 -5
- package/engine/encoding.mjs +28 -22
- package/engine/heading-attributes.mjs +2 -5
- package/engine/language-writings.mjs +34 -1
- package/engine/markdown-blocks.mjs +58 -0
- package/package.json +1 -1
- package/types/engine/content-index.d.mts +28 -2
- package/types/engine/content-session.d.mts +1 -0
- package/types/engine/content-timings.d.mts +7 -0
- package/types/engine/encoding-lexicon.d.mts +6 -35
- package/types/engine/encoding-template.d.mts +1 -0
- package/types/engine/encoding-terms.d.mts +42 -4
- package/types/engine/encoding.d.mts +1 -1
- package/types/engine/language-writings.d.mts +3 -1
- package/types/engine/markdown-blocks.d.mts +8 -0
|
@@ -20,7 +20,7 @@ import { isNoteRecord } from "./index-records.mjs";
|
|
|
20
20
|
import { nearest } from "./near-miss.mjs";
|
|
21
21
|
import { isLanguageRecord, lineageChain, noteAddressOf } from "./word-rows.mjs";
|
|
22
22
|
import { derivedForms, foldWord } from "./word-forms.mjs";
|
|
23
|
-
import { WRITTEN_ENDINGS } from "./language-terms.mjs";
|
|
23
|
+
import { WRITTEN_ENDINGS, RESERVED_TONGUE } from "./language-terms.mjs";
|
|
24
24
|
import { LANGUAGE_SUBTYPE, glossText } from "./word-terms.mjs";
|
|
25
25
|
import { writingsOf } from "./language-writings.mjs";
|
|
26
26
|
import { encodeText, validateSource, EncodingError } from "./encoding.mjs";
|
|
@@ -389,9 +389,21 @@ export function nativeRows(records) {
|
|
|
389
389
|
return rows;
|
|
390
390
|
}
|
|
391
391
|
|
|
392
|
-
/**
|
|
392
|
+
/** Each lexicon's projection, made once and shared by every conversion that reads it. */
|
|
393
|
+
const serialized = new WeakMap();
|
|
394
|
+
|
|
395
|
+
/**
|
|
396
|
+
* Project only conversion-readable language data to a JSON-safe object. The
|
|
397
|
+
* projection of one lexicon is made once and shared, so it must not be modified.
|
|
398
|
+
*/
|
|
393
399
|
export function serializeLexicon(lexicon) {
|
|
394
400
|
if (!lexicon) return undefined;
|
|
401
|
+
if (!serialized.has(lexicon)) serialized.set(lexicon, projectLexicon(lexicon));
|
|
402
|
+
return serialized.get(lexicon);
|
|
403
|
+
}
|
|
404
|
+
|
|
405
|
+
/** The conversion-readable data of a lexicon, copied. */
|
|
406
|
+
function projectLexicon(lexicon) {
|
|
395
407
|
const select = (value, keys) =>
|
|
396
408
|
Object.fromEntries(
|
|
397
409
|
keys
|
|
@@ -658,7 +670,9 @@ export function checkEncodingLexicon(note, ctx) {
|
|
|
658
670
|
);
|
|
659
671
|
listOf(element?.examples).forEach((form, j) => add(form, ["elements", i, "examples", j]));
|
|
660
672
|
});
|
|
661
|
-
listOf(data.register).forEach((name, i) =>
|
|
673
|
+
listOf(data.register).forEach((name, i) => {
|
|
674
|
+
if (name?.tongue !== RESERVED_TONGUE) add(name?.name, ["register", i, "name"]);
|
|
675
|
+
});
|
|
662
676
|
const findings = [],
|
|
663
677
|
seen = new Set();
|
|
664
678
|
for (const encoding of encodings)
|
|
@@ -61,10 +61,13 @@ export function templateProgram(data, options, fault) {
|
|
|
61
61
|
"A glyph code must resolve to nonempty Unicode scalar text",
|
|
62
62
|
);
|
|
63
63
|
}
|
|
64
|
-
const resolve = (value, key) => {
|
|
64
|
+
const resolve = (value, key, allowEmpty = false) => {
|
|
65
65
|
const sequence = typeof value === "string" ? [value] : value;
|
|
66
|
-
if (!Array.isArray(sequence) || !sequence.length)
|
|
67
|
-
fault(
|
|
66
|
+
if (!Array.isArray(sequence) || (!sequence.length && !allowEmpty))
|
|
67
|
+
fault(
|
|
68
|
+
key,
|
|
69
|
+
"A mapping is a glyph code or a code sequence; an empty sequence requires canonical round trips",
|
|
70
|
+
);
|
|
68
71
|
return sequence
|
|
69
72
|
.map((name) => {
|
|
70
73
|
if (
|
|
@@ -82,6 +85,8 @@ export function templateProgram(data, options, fault) {
|
|
|
82
85
|
[
|
|
83
86
|
"normalization",
|
|
84
87
|
"matching",
|
|
88
|
+
"roundTrip",
|
|
89
|
+
"word-separators",
|
|
85
90
|
"case",
|
|
86
91
|
"stress",
|
|
87
92
|
"vowel-groups",
|
|
@@ -95,6 +100,9 @@ export function templateProgram(data, options, fault) {
|
|
|
95
100
|
const rules = data.rules;
|
|
96
101
|
exact(rules.normalization, "NFC", "rules.normalization");
|
|
97
102
|
exact(rules.matching, "longest-first", "rules.matching");
|
|
103
|
+
if (rules.roundTrip !== undefined && !["exact", "canonical"].includes(rules.roundTrip))
|
|
104
|
+
fault("rules.roundTrip", "roundTrip is exact or canonical");
|
|
105
|
+
const canonical = rules.roundTrip === "canonical";
|
|
98
106
|
if (rules.case !== undefined)
|
|
99
107
|
fixed(
|
|
100
108
|
rules.case,
|
|
@@ -106,7 +114,11 @@ export function templateProgram(data, options, fault) {
|
|
|
106
114
|
"rules.case",
|
|
107
115
|
);
|
|
108
116
|
for (const key of ["diaeresis", "grave-accent"])
|
|
109
|
-
if (rules[key] !== undefined)
|
|
117
|
+
if (rules[key] !== undefined) {
|
|
118
|
+
closed(rules[key], ["allowed"], `rules.${key}`);
|
|
119
|
+
if (typeof rules[key].allowed !== "boolean")
|
|
120
|
+
fault(`rules.${key}.allowed`, "allowed must be true or false");
|
|
121
|
+
}
|
|
110
122
|
const vowels = data.vowels ?? "";
|
|
111
123
|
if (!scalar(vowels)) fault("vowels", "vowels must be a Unicode scalar string");
|
|
112
124
|
const vowelSet = new Set([...vowels].map(base));
|
|
@@ -189,8 +201,9 @@ export function templateProgram(data, options, fault) {
|
|
|
189
201
|
if (/[\p{White_Space}⟦⟧]/u.test(from))
|
|
190
202
|
fault(key, "Whitespace and literal markers belong to the shared standard");
|
|
191
203
|
const decomposed = from.normalize("NFD");
|
|
192
|
-
if (rules.diaeresis && decomposed.includes("\u0308"))
|
|
193
|
-
|
|
204
|
+
if (rules.diaeresis?.allowed === false && decomposed.includes("\u0308"))
|
|
205
|
+
fault(key, "Diaeresis is forbidden");
|
|
206
|
+
if (rules["grave-accent"]?.allowed === false && decomposed.includes("\u0300"))
|
|
194
207
|
fault(key, "Grave accents are forbidden");
|
|
195
208
|
};
|
|
196
209
|
const validateOutput = (from, to, key) => {
|
|
@@ -215,10 +228,36 @@ export function templateProgram(data, options, fault) {
|
|
|
215
228
|
for (const [from, output] of Object.entries(data.mappings)) {
|
|
216
229
|
const key = `mappings.${from}`;
|
|
217
230
|
validateSource(from, key);
|
|
218
|
-
const to = resolve(output, key);
|
|
231
|
+
const to = resolve(output, key, canonical);
|
|
219
232
|
validateOutput(from, to, key);
|
|
220
233
|
mappings.push([from, to]);
|
|
221
234
|
}
|
|
235
|
+
const separators = [];
|
|
236
|
+
if (rules["word-separators"] !== undefined) {
|
|
237
|
+
if (!Array.isArray(rules["word-separators"]))
|
|
238
|
+
fault("rules.word-separators", "word-separators must be a list");
|
|
239
|
+
const seen = new Set();
|
|
240
|
+
for (const [i, from] of rules["word-separators"].entries()) {
|
|
241
|
+
const key = `rules.word-separators.${i}`;
|
|
242
|
+
validateSource(from, key);
|
|
243
|
+
if (
|
|
244
|
+
[...from].some(
|
|
245
|
+
(char) =>
|
|
246
|
+
"^/{}".includes(char) || /[\p{Cc}\p{Cf}\u{E0100}-\u{E01EF}]/u.test(char),
|
|
247
|
+
) ||
|
|
248
|
+
STANDARD_ROWS.some((row) => (Array.isArray(row) ? row[0] : row.from) === from)
|
|
249
|
+
)
|
|
250
|
+
fault(
|
|
251
|
+
key,
|
|
252
|
+
"Word separators cannot replace shared punctuation or contain structural controls",
|
|
253
|
+
);
|
|
254
|
+
if (seen.has(from)) fault(key, "A word separator is stated twice");
|
|
255
|
+
if (Object.hasOwn(data.mappings, from))
|
|
256
|
+
fault(key, "A word separator cannot also have an ordinary mapping");
|
|
257
|
+
seen.add(from);
|
|
258
|
+
separators.push({ from, to: STANDARD_CONTROLS.wordSeparator });
|
|
259
|
+
}
|
|
260
|
+
}
|
|
222
261
|
const contextual = [];
|
|
223
262
|
if (rules["contextual-mappings"] !== undefined) {
|
|
224
263
|
if (!Array.isArray(rules["contextual-mappings"]))
|
|
@@ -226,21 +265,36 @@ export function templateProgram(data, options, fault) {
|
|
|
226
265
|
const seen = new Set();
|
|
227
266
|
for (const [i, row] of rules["contextual-mappings"].entries()) {
|
|
228
267
|
const key = `rules.contextual-mappings.${i}`;
|
|
229
|
-
closed(row, ["match", "before", "emit"], key);
|
|
230
|
-
if (
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
if (
|
|
268
|
+
closed(row, ["match", "before", "at", "emit"], key);
|
|
269
|
+
if (!scalar(row.match) || !row.match || row.match !== row.match.normalize("NFC"))
|
|
270
|
+
fault(`${key}.match`, "A contextual match must be nonempty NFC text");
|
|
271
|
+
if (row.before === undefined && row.at === undefined)
|
|
272
|
+
fault(key, "A contextual mapping requires before or at");
|
|
273
|
+
if (row.before !== undefined && !["vowel", "consonant"].includes(row.before))
|
|
274
|
+
fault(`${key}.before`, "before is vowel or consonant");
|
|
275
|
+
if (row.at !== undefined && !["start", "end", "line-start"].includes(row.at))
|
|
276
|
+
fault(`${key}.at`, "at is start, end or line-start");
|
|
277
|
+
if (row.before && !vowels)
|
|
278
|
+
fault(
|
|
279
|
+
`${key}.before`,
|
|
280
|
+
"Before-vowel or before-consonant mappings require declared vowels",
|
|
281
|
+
);
|
|
282
|
+
const condition = JSON.stringify([row.match, row.before ?? null, row.at ?? null]);
|
|
283
|
+
if (seen.has(condition))
|
|
284
|
+
fault(
|
|
285
|
+
`${key}.match`,
|
|
286
|
+
"A contextual mapping repeats the same source and conditions",
|
|
287
|
+
);
|
|
239
288
|
validateSource(row.match, `${key}.match`);
|
|
240
|
-
const to = resolve(row.emit, `${key}.emit
|
|
289
|
+
const to = resolve(row.emit, `${key}.emit`, canonical);
|
|
241
290
|
validateOutput(row.match, to, `${key}.emit`);
|
|
242
|
-
seen.add(
|
|
243
|
-
contextual.push({
|
|
291
|
+
seen.add(condition);
|
|
292
|
+
contextual.push({
|
|
293
|
+
from: row.match,
|
|
294
|
+
to,
|
|
295
|
+
...(row.before !== undefined ? { before: row.before } : {}),
|
|
296
|
+
...(row.at !== undefined ? { at: row.at } : {}),
|
|
297
|
+
});
|
|
244
298
|
}
|
|
245
299
|
}
|
|
246
300
|
if (rules.case)
|
|
@@ -256,26 +310,32 @@ export function templateProgram(data, options, fault) {
|
|
|
256
310
|
...STANDARD_ROWS.filter((row) => !overridden.has(Array.isArray(row) ? row[0] : row.from)),
|
|
257
311
|
...contextual,
|
|
258
312
|
...mappings,
|
|
313
|
+
...separators,
|
|
259
314
|
];
|
|
260
|
-
|
|
261
|
-
[
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
315
|
+
const sourceKey = (row) => {
|
|
316
|
+
if (Array.isArray(row)) return `mappings.${row[0]}`;
|
|
317
|
+
const separator = separators.indexOf(row);
|
|
318
|
+
if (separator >= 0) return `rules.word-separators.${separator}`;
|
|
319
|
+
const context = contextual.indexOf(row);
|
|
320
|
+
return context < 0 ? "rules" : `rules.contextual-mappings.${context}`;
|
|
321
|
+
};
|
|
322
|
+
if (!canonical)
|
|
323
|
+
assertUniqueOutputs(
|
|
324
|
+
[
|
|
325
|
+
...rows.map((row) =>
|
|
326
|
+
Array.isArray(row) ?
|
|
327
|
+
{ from: row[0], to: row[1], key: sourceKey(row) }
|
|
328
|
+
: { ...row, key: sourceKey(row) },
|
|
329
|
+
),
|
|
330
|
+
{ from: " ", to: " ", key: "rules" },
|
|
331
|
+
],
|
|
332
|
+
fault,
|
|
333
|
+
);
|
|
275
334
|
const accepts = data.accepts ?? [
|
|
276
335
|
...new Set([
|
|
277
336
|
...Object.keys(data.mappings).join(""),
|
|
278
337
|
...contextual.map((r) => r.from).join(""),
|
|
338
|
+
...separators.map((row) => row.from).join(""),
|
|
279
339
|
]),
|
|
280
340
|
];
|
|
281
341
|
if (!Array.isArray(accepts)) fault("accepts", "accepts must be a list of characters or ranges");
|
|
@@ -285,17 +345,14 @@ export function templateProgram(data, options, fault) {
|
|
|
285
345
|
fonts: [data.font],
|
|
286
346
|
accepts: [...accepts, ...STANDARD_ROWS.map((r) => (Array.isArray(r) ? r[0] : r.from))],
|
|
287
347
|
separator: " ",
|
|
348
|
+
roundTrip: canonical ? "canonical" : "exact",
|
|
288
349
|
vowels,
|
|
289
350
|
rows,
|
|
290
351
|
codes: Object.entries(options.codes),
|
|
291
352
|
controls: STANDARD_CONTROLS,
|
|
292
353
|
...(rules.case ? { groupCase: "initial" } : {}),
|
|
293
354
|
},
|
|
294
|
-
rowKeys: rows.map(
|
|
295
|
-
if (Array.isArray(row)) return `mappings.${row[0]}`;
|
|
296
|
-
const index = contextual.indexOf(row);
|
|
297
|
-
return index < 0 ? "rules" : `rules.contextual-mappings.${index}`;
|
|
298
|
-
}),
|
|
355
|
+
rowKeys: rows.map(sourceKey),
|
|
299
356
|
metadata: {
|
|
300
357
|
name: options.name ?? `${data.language}-${data.font}`,
|
|
301
358
|
language: data.language,
|
|
@@ -305,8 +362,8 @@ export function templateProgram(data, options, fault) {
|
|
|
305
362
|
vowelGroups: Object.freeze(vowelGroups),
|
|
306
363
|
maxVowelGroup: Math.max(0, ...vowelGroups.map((group) => [...group].length)),
|
|
307
364
|
stress,
|
|
308
|
-
diaeresis:
|
|
309
|
-
grave:
|
|
365
|
+
diaeresis: rules.diaeresis?.allowed === false,
|
|
366
|
+
grave: rules["grave-accent"]?.allowed === false,
|
|
310
367
|
}),
|
|
311
368
|
},
|
|
312
369
|
};
|
|
@@ -28,7 +28,7 @@ const fixed = (name, value) => ({
|
|
|
28
28
|
const block = (name, describe, fields) => ({ name, kind: "map", shape: "a map", describe, fields });
|
|
29
29
|
const sequence = {
|
|
30
30
|
kind: "string-or-list",
|
|
31
|
-
shape: "a glyph code or list of codes",
|
|
31
|
+
shape: "a glyph code or list of codes (empty only with canonical round trips)",
|
|
32
32
|
entries: {
|
|
33
33
|
kind: "string",
|
|
34
34
|
shape: "a glyph code",
|
|
@@ -39,6 +39,25 @@ const sequence = {
|
|
|
39
39
|
export const ENCODING_RULE_FIELDS = Object.freeze([
|
|
40
40
|
fixed("normalization", "NFC"),
|
|
41
41
|
fixed("matching", "longest-first"),
|
|
42
|
+
{
|
|
43
|
+
...text(
|
|
44
|
+
"roundTrip",
|
|
45
|
+
"Exact by default; canonical permits aliases and empty code sequences and decodes an authored conventional spelling.",
|
|
46
|
+
),
|
|
47
|
+
allowed: ["exact", "canonical"],
|
|
48
|
+
},
|
|
49
|
+
{
|
|
50
|
+
name: "word-separators",
|
|
51
|
+
kind: "list",
|
|
52
|
+
shape: "a list of nonempty NFC source sequences",
|
|
53
|
+
entries: {
|
|
54
|
+
kind: "string",
|
|
55
|
+
shape: "a word separator",
|
|
56
|
+
describe: "A source sequence that writes the shared word space.",
|
|
57
|
+
},
|
|
58
|
+
describe:
|
|
59
|
+
"Source word dividers written as shared spaces, after contextual alternatives; cannot also be ordinary mappings or replace shared punctuation.",
|
|
60
|
+
},
|
|
42
61
|
block("case", "Case handling for letters and consonant groups.", [
|
|
43
62
|
fixed("single-letter", "preserve"),
|
|
44
63
|
fixed("consonant-group", "lowercase-or-initial-uppercase"),
|
|
@@ -70,19 +89,37 @@ export const ENCODING_RULE_FIELDS = Object.freeze([
|
|
|
70
89
|
describe: "The complete authored group inventory.",
|
|
71
90
|
},
|
|
72
91
|
]),
|
|
73
|
-
|
|
74
|
-
|
|
92
|
+
...["diaeresis", "grave-accent"].map((name) =>
|
|
93
|
+
block(name, `Whether ${name} mappings are allowed.`, [
|
|
94
|
+
{
|
|
95
|
+
name: "allowed",
|
|
96
|
+
kind: "boolean",
|
|
97
|
+
shape: "true or false",
|
|
98
|
+
required: true,
|
|
99
|
+
describe:
|
|
100
|
+
"False forbids the accent; true permits explicitly mapped and accepted characters.",
|
|
101
|
+
},
|
|
102
|
+
]),
|
|
103
|
+
),
|
|
75
104
|
{
|
|
76
105
|
name: "contextual-mappings",
|
|
77
106
|
kind: "list",
|
|
78
107
|
shape: "a list of conditional mappings",
|
|
79
|
-
describe:
|
|
108
|
+
describe:
|
|
109
|
+
"Alternatives conditioned on the following letter or source position, applied before unconditional mappings.",
|
|
80
110
|
entries: {
|
|
81
111
|
kind: "map",
|
|
82
112
|
shape: "a contextual mapping",
|
|
83
113
|
fields: [
|
|
84
114
|
text("match", "The source sequence.", true),
|
|
85
|
-
|
|
115
|
+
{
|
|
116
|
+
...text("before", "Match before a declared vowel or another letter."),
|
|
117
|
+
allowed: ["vowel", "consonant"],
|
|
118
|
+
},
|
|
119
|
+
{
|
|
120
|
+
...text("at", "Match at word start, word end or line start."),
|
|
121
|
+
allowed: ["start", "end", "line-start"],
|
|
122
|
+
},
|
|
86
123
|
{
|
|
87
124
|
name: "emit",
|
|
88
125
|
...sequence,
|
package/engine/encoding.mjs
CHANGED
|
@@ -322,15 +322,13 @@ function boundary(char) {
|
|
|
322
322
|
// Python str.isspace also includes the four information separators.
|
|
323
323
|
return SPACE.test(char) || /[\u001C-\u001F]/u.test(char) || BOUNDARY.test(char);
|
|
324
324
|
}
|
|
325
|
+
function beforeHolds(condition, char, vowels) {
|
|
326
|
+
return Boolean(
|
|
327
|
+
char && (condition === "vowel" ? vowels.has(char) : !vowels.has(char) && LETTER.test(char)),
|
|
328
|
+
);
|
|
329
|
+
}
|
|
325
330
|
function holds(row, text, start, end, vowels) {
|
|
326
|
-
if (row.before)
|
|
327
|
-
const char = firstAt(text, end);
|
|
328
|
-
if (
|
|
329
|
-
!char ||
|
|
330
|
-
(row.before === "vowel" ? !vowels.has(char) : vowels.has(char) || !LETTER.test(char))
|
|
331
|
-
)
|
|
332
|
-
return false;
|
|
333
|
-
}
|
|
331
|
+
if (row.before && !beforeHolds(row.before, firstAt(text, end), vowels)) return false;
|
|
334
332
|
if (row.at === "line-start") {
|
|
335
333
|
const prefix = text.slice(
|
|
336
334
|
Math.max(text.lastIndexOf("\n", start - 1), text.lastIndexOf("/", start - 1)) + 1,
|
|
@@ -834,7 +832,7 @@ function decodeRawText(enc, text, { font } = {}) {
|
|
|
834
832
|
}
|
|
835
833
|
|
|
836
834
|
/**
|
|
837
|
-
* Decode
|
|
835
|
+
* Decode exact output or the conventional reading of an explicitly canonical note.
|
|
838
836
|
* @param {ReturnType<typeof compileEncoding>} enc - Compiled encoding note.
|
|
839
837
|
* @param {string} text - Native text with optional literal spans.
|
|
840
838
|
* @param {{font?: string}} [options] - Selected encoding face.
|
|
@@ -845,17 +843,19 @@ export function decodeText(enc, text, options = {}) {
|
|
|
845
843
|
const readings = tiers(enc, options.font)[0].readings;
|
|
846
844
|
if (typeof text !== "string") throw new EncodingError("Text must be a string");
|
|
847
845
|
if (!templateReaders.has(enc)) {
|
|
848
|
-
const root = { children: new Map(), row: undefined };
|
|
849
|
-
|
|
850
|
-
|
|
851
|
-
|
|
852
|
-
|
|
853
|
-
|
|
854
|
-
|
|
855
|
-
node
|
|
856
|
-
|
|
857
|
-
node.row = row;
|
|
846
|
+
const root = { children: new Map(), row: undefined, rows: [] };
|
|
847
|
+
const rows = new Set([...readings.values()].flat().concat(enc.shared.rows));
|
|
848
|
+
for (const row of rows) {
|
|
849
|
+
if (!row.to) continue;
|
|
850
|
+
let node = root;
|
|
851
|
+
for (const char of row.to) {
|
|
852
|
+
if (!node.children.has(char))
|
|
853
|
+
node.children.set(char, { children: new Map(), row: undefined, rows: [] });
|
|
854
|
+
node = node.children.get(char);
|
|
858
855
|
}
|
|
856
|
+
node.row ??= row;
|
|
857
|
+
node.rows.push(row);
|
|
858
|
+
}
|
|
859
859
|
templateReaders.set(enc, root);
|
|
860
860
|
}
|
|
861
861
|
const reader = templateReaders.get(enc);
|
|
@@ -888,9 +888,15 @@ export function decodeText(enc, text, options = {}) {
|
|
|
888
888
|
cursor = cursor.children.get(char);
|
|
889
889
|
if (!cursor) break;
|
|
890
890
|
next += char.length;
|
|
891
|
-
|
|
892
|
-
|
|
893
|
-
|
|
891
|
+
if (!cursor.row || !paths.has(next)) continue;
|
|
892
|
+
const following = firstAt(paths.get(next).from, 0);
|
|
893
|
+
const preferred = cursor.rows.find(
|
|
894
|
+
(row) => !row.before || beforeHolds(row.before, following, enc.vowels),
|
|
895
|
+
);
|
|
896
|
+
// Canonical source can omit the vowel or marker that selected a form.
|
|
897
|
+
// Prefer applicable aliases, then retain the first authored reading.
|
|
898
|
+
const row = preferred ?? (enc.roundTrip === "canonical" ? cursor.row : undefined);
|
|
899
|
+
if (!row) {
|
|
894
900
|
failedContext = Math.min(failedContext, at);
|
|
895
901
|
continue;
|
|
896
902
|
}
|
|
@@ -34,15 +34,12 @@
|
|
|
34
34
|
*/
|
|
35
35
|
|
|
36
36
|
import { parseExtensionAttributes, refusedAttributes } from "./extension-attributes.mjs";
|
|
37
|
-
import
|
|
38
|
-
|
|
39
|
-
/** Literal block ranges use the renderer's Markdown grammar. */
|
|
40
|
-
const structureParser = new MarkdownIt({ html: true });
|
|
37
|
+
import { blockTokens } from "./markdown-blocks.mjs";
|
|
41
38
|
|
|
42
39
|
/** Body lines occupied by code or raw HTML, including comment examples. */
|
|
43
40
|
function literalHeadingLines(source) {
|
|
44
41
|
const lines = new Set();
|
|
45
|
-
for (const token of
|
|
42
|
+
for (const token of blockTokens(source)) {
|
|
46
43
|
if (!token.map || !["fence", "code_block", "html_block"].includes(token.type)) continue;
|
|
47
44
|
for (let line = token.map[0]; line < token.map[1]; line++) lines.add(line);
|
|
48
45
|
}
|
|
@@ -80,8 +80,13 @@ function oneOf(form, candidates) {
|
|
|
80
80
|
return { ambiguous: { form, glosses: candidates.map(({ gloss }) => gloss) } };
|
|
81
81
|
}
|
|
82
82
|
|
|
83
|
+
/** Each language data's writings, by encoding and face. */
|
|
84
|
+
const writingsFound = new WeakMap();
|
|
85
|
+
|
|
83
86
|
/**
|
|
84
|
-
* The writings of a language's words under one encoding in one face.
|
|
87
|
+
* The writings of a language's words under one encoding in one face. One
|
|
88
|
+
* `data` yields one set of writings per encoding and face, and each word's
|
|
89
|
+
* writing is found once, so `data` must not be modified once it is read here.
|
|
85
90
|
*
|
|
86
91
|
* @param {Record<string, any>|undefined} data - The language note's `data`.
|
|
87
92
|
* @param {string} enc - The encoding's name.
|
|
@@ -90,6 +95,34 @@ function oneOf(form, candidates) {
|
|
|
90
95
|
* The writing of a run's word, and of the `data.words` entry at an index.
|
|
91
96
|
*/
|
|
92
97
|
export function writingsOf(data, enc, font) {
|
|
98
|
+
if (!isMap(data)) return findWritings(data, enc, font);
|
|
99
|
+
if (!writingsFound.has(data)) writingsFound.set(data, new Map());
|
|
100
|
+
const byFace = writingsFound.get(data);
|
|
101
|
+
const key = JSON.stringify([enc, font ?? null]);
|
|
102
|
+
if (!byFace.has(key)) {
|
|
103
|
+
const found = findWritings(data, enc, font);
|
|
104
|
+
const words = new Map();
|
|
105
|
+
byFace.set(key, {
|
|
106
|
+
word: (written) => {
|
|
107
|
+
if (!words.has(written)) words.set(written, found.word(written));
|
|
108
|
+
return words.get(written);
|
|
109
|
+
},
|
|
110
|
+
entry: found.entry,
|
|
111
|
+
});
|
|
112
|
+
}
|
|
113
|
+
return byFace.get(key);
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
/**
|
|
117
|
+
* The writings of a language's words under one encoding in one face, found
|
|
118
|
+
* afresh; see {@link writingsOf}.
|
|
119
|
+
*
|
|
120
|
+
* @param {Record<string, any>|undefined} data - The language note's `data`.
|
|
121
|
+
* @param {string} enc - The encoding's name.
|
|
122
|
+
* @param {string|undefined} font - The face.
|
|
123
|
+
* @returns {{word: (word: string) => Writing, entry: (index: number) => Writing}}
|
|
124
|
+
*/
|
|
125
|
+
function findWritings(data, enc, font) {
|
|
93
126
|
const words = listOf(data?.words);
|
|
94
127
|
const affixes = listOf(data?.affixes).filter(isMap);
|
|
95
128
|
const grammar = isMap(data?.grammar) ? data.grammar : undefined;
|
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
/*
|
|
2
|
+
* This file is part of the Song of Heroic Lands (SoHL) system for Foundry VTT.
|
|
3
|
+
* Copyright (c) 2024-2026 Tom Rodriguez ("Toasty") — <toasty@heroiclands.org>
|
|
4
|
+
*
|
|
5
|
+
* This work is licensed under the GNU General Public License v3.0 (GPLv3).
|
|
6
|
+
* You may copy, modify, and distribute it under the terms of that license.
|
|
7
|
+
*
|
|
8
|
+
* For full terms, see the LICENSE.md file in the project root or visit:
|
|
9
|
+
* https://www.gnu.org/licenses/gpl-3.0.html
|
|
10
|
+
*
|
|
11
|
+
* SPDX-License-Identifier: GPL-3.0-or-later
|
|
12
|
+
*/
|
|
13
|
+
|
|
14
|
+
/**
|
|
15
|
+
* The block structure of a Markdown source, in the renderer's grammar.
|
|
16
|
+
*
|
|
17
|
+
* Several scanners ask only where a source's fences, code blocks, raw HTML and
|
|
18
|
+
* quotes lie, each reading block tokens' `type`, `map` and `level` and nothing
|
|
19
|
+
* else. Inline parsing fills only an `inline` token's `children`, so this
|
|
20
|
+
* parser skips it, and the block tokens it yields are the ones a full parse
|
|
21
|
+
* yields. The scanners commonly ask about the same source in turn, so the most
|
|
22
|
+
* recent sources' tokens are kept.
|
|
23
|
+
*
|
|
24
|
+
* @module
|
|
25
|
+
*/
|
|
26
|
+
|
|
27
|
+
import MarkdownIt from "markdown-it";
|
|
28
|
+
|
|
29
|
+
/** The renderer's block grammar, with inline parsing and its passes switched off. */
|
|
30
|
+
const parser = new MarkdownIt({ html: true });
|
|
31
|
+
parser.core.ruler.disable(["inline", "linkify", "replacements", "smartquotes", "text_join"]);
|
|
32
|
+
|
|
33
|
+
/** How many recent sources keep their tokens. */
|
|
34
|
+
const RECENT = 16;
|
|
35
|
+
|
|
36
|
+
/** @type {Map<string, import("markdown-it").Token[]>} */
|
|
37
|
+
const recent = new Map();
|
|
38
|
+
|
|
39
|
+
/**
|
|
40
|
+
* The block tokens of a Markdown source. The array is shared between callers
|
|
41
|
+
* and must not be modified; an `inline` token carries no `children`.
|
|
42
|
+
*
|
|
43
|
+
* @param {unknown} source - Markdown.
|
|
44
|
+
* @returns {readonly import("markdown-it").Token[]} The block tokens, in order.
|
|
45
|
+
*/
|
|
46
|
+
export function blockTokens(source) {
|
|
47
|
+
const text = String(source ?? "");
|
|
48
|
+
const hit = recent.get(text);
|
|
49
|
+
if (hit) {
|
|
50
|
+
recent.delete(text);
|
|
51
|
+
recent.set(text, hit);
|
|
52
|
+
return hit;
|
|
53
|
+
}
|
|
54
|
+
const tokens = parser.parse(text, {});
|
|
55
|
+
recent.set(text, tokens);
|
|
56
|
+
if (recent.size > RECENT) recent.delete(recent.keys().next().value);
|
|
57
|
+
return tokens;
|
|
58
|
+
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@heroiclands/package-build",
|
|
3
|
-
"version": "25.1.
|
|
3
|
+
"version": "25.1.2",
|
|
4
4
|
"description": "Shared toolchain for building and shipping a HeroicLands Foundry VTT package \u2014 content compilation, manifest, localization, staging, bundle, release and deployment.",
|
|
5
5
|
"license": "GPL-3.0-or-later",
|
|
6
6
|
"type": "module",
|
|
@@ -225,10 +225,12 @@ export function resolveCultures(records: Array<Record<string, any>>, { contentPa
|
|
|
225
225
|
* @param {Map<string, {raw: string}>} [options.sources] - Selected captured source strings.
|
|
226
226
|
* @param {import("./content-session.mjs").ContentSyntax} [options.syntax] - Session scanner projections.
|
|
227
227
|
* @param {Iterable<string>} [options.nameTypes] - Additional bare name kinds from configured type declarations.
|
|
228
|
+
* @param {ReturnType<typeof import("./content-timings.mjs").createContentTimings>} [options.timings] -
|
|
229
|
+
* Invocation stage accounting, which records each phase of the derivation.
|
|
228
230
|
* @returns {Array<Record<string, any>>} The records, in a total order that does
|
|
229
231
|
* not depend on directory-read order.
|
|
230
232
|
*/
|
|
231
|
-
export function collectContentIndex(contentBase: string, { contentPackage, currentDate, defaultCalendar, configFile, rootDir, languages, skipDirectories, assetsBase, manifest, problems, addressContext, foreignRecord, parsed, sources, syntax, nameTypes, }: {
|
|
233
|
+
export function collectContentIndex(contentBase: string, { contentPackage, currentDate, defaultCalendar, configFile, rootDir, languages, skipDirectories, assetsBase, manifest, problems, addressContext, foreignRecord, parsed, sources, syntax, nameTypes, timings, }: {
|
|
232
234
|
contentPackage: string;
|
|
233
235
|
skipDirectories: readonly string[];
|
|
234
236
|
assetsBase?: string | undefined;
|
|
@@ -257,6 +259,17 @@ export function collectContentIndex(contentBase: string, { contentPackage, curre
|
|
|
257
259
|
}> | undefined;
|
|
258
260
|
syntax?: import("./content-session.mjs").ContentSyntax | undefined;
|
|
259
261
|
nameTypes?: Iterable<string> | undefined;
|
|
262
|
+
timings?: {
|
|
263
|
+
sync<T>(name: string, work: () => T): T;
|
|
264
|
+
measure<T>(name: string, work: () => Promise<T> | T): Promise<T>;
|
|
265
|
+
start(name: string): () => void;
|
|
266
|
+
hit(name: string): void;
|
|
267
|
+
snapshot(): {
|
|
268
|
+
totalMs: number;
|
|
269
|
+
stages: any;
|
|
270
|
+
hits: any;
|
|
271
|
+
};
|
|
272
|
+
} | undefined;
|
|
260
273
|
}): Array<Record<string, any>>;
|
|
261
274
|
/**
|
|
262
275
|
* Serialize records as JSON Lines.
|
|
@@ -301,9 +314,11 @@ export function serializeContentIndex(records: Array<Record<string, any>>): stri
|
|
|
301
314
|
* @param {import("./content-session.mjs").ContentSyntax} [opts.syntax] - Session scanner projections.
|
|
302
315
|
* @param {Iterable<string>} [opts.nameTypes] - Bare name kinds, defaulting to configured item types.
|
|
303
316
|
* @param {Map<string, object>} [opts.foreignIndex] - Captured dependency address view.
|
|
317
|
+
* @param {ReturnType<typeof import("./content-timings.mjs").createContentTimings>} [opts.timings] -
|
|
318
|
+
* Invocation stage accounting.
|
|
304
319
|
* @returns {object[]} One record per note, plus one per documentation entry.
|
|
305
320
|
*/
|
|
306
|
-
export function indexRecordsFor({ contentBase, assetsBase, config, skipDirectories, problems, parsed, sources, syntax, nameTypes, foreignIndex, }?: {
|
|
321
|
+
export function indexRecordsFor({ contentBase, assetsBase, config, skipDirectories, problems, parsed, sources, syntax, nameTypes, foreignIndex, timings, }?: {
|
|
307
322
|
contentBase?: string | undefined;
|
|
308
323
|
assetsBase?: string | undefined;
|
|
309
324
|
config?: object | undefined;
|
|
@@ -322,6 +337,17 @@ export function indexRecordsFor({ contentBase, assetsBase, config, skipDirectori
|
|
|
322
337
|
syntax?: import("./content-session.mjs").ContentSyntax | undefined;
|
|
323
338
|
nameTypes?: Iterable<string> | undefined;
|
|
324
339
|
foreignIndex?: Map<string, object> | undefined;
|
|
340
|
+
timings?: {
|
|
341
|
+
sync<T>(name: string, work: () => T): T;
|
|
342
|
+
measure<T>(name: string, work: () => Promise<T> | T): Promise<T>;
|
|
343
|
+
start(name: string): () => void;
|
|
344
|
+
hit(name: string): void;
|
|
345
|
+
snapshot(): {
|
|
346
|
+
totalMs: number;
|
|
347
|
+
stages: any;
|
|
348
|
+
hits: any;
|
|
349
|
+
};
|
|
350
|
+
} | undefined;
|
|
325
351
|
}): object[];
|
|
326
352
|
/**
|
|
327
353
|
* Prepare the complete bodies a content-index publication describes.
|
|
@@ -27,6 +27,7 @@ export function createContentSession({ config, contentBase, assetsBase, skipDire
|
|
|
27
27
|
timings?: {
|
|
28
28
|
sync<T>(name: string, work: () => T): T;
|
|
29
29
|
measure<T>(name: string, work: () => Promise<T> | T): Promise<T>;
|
|
30
|
+
start(name: string): () => void;
|
|
30
31
|
hit(name: string): void;
|
|
31
32
|
snapshot(): {
|
|
32
33
|
totalMs: number;
|
|
@@ -4,6 +4,13 @@ export function createContentTimings(): {
|
|
|
4
4
|
sync<T>(name: string, work: () => T): T;
|
|
5
5
|
/** @template T @param {string} name @param {() => Promise<T>|T} work @returns {Promise<T>} */
|
|
6
6
|
measure<T>(name: string, work: () => Promise<T> | T): Promise<T>;
|
|
7
|
+
/**
|
|
8
|
+
* Open a stage that runs until the returned function is called, or
|
|
9
|
+
* until the stage it was opened in ends.
|
|
10
|
+
* @param {string} name
|
|
11
|
+
* @returns {() => void}
|
|
12
|
+
*/
|
|
13
|
+
start(name: string): () => void;
|
|
7
14
|
/** @param {string} name */
|
|
8
15
|
hit(name: string): void;
|
|
9
16
|
snapshot(): {
|