@heroiclands/package-build 25.1.1 → 25.1.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +14 -0
- package/bin/content-commands.mjs +43 -6
- package/docs/encoding-core.md +11 -0
- package/docs/reference/format-details.md +30 -2
- package/docs/reference/note-types.md +258 -257
- package/engine/content-alerts.mjs +2 -2
- package/engine/content-blocks.mjs +2 -2
- package/engine/content-index.mjs +23 -1
- package/engine/content-scripts.mjs +15 -6
- package/engine/content-scroll.mjs +2 -4
- package/engine/content-session.mjs +1 -0
- package/engine/content-timings.mjs +13 -1
- package/engine/content-verse.mjs +2 -2
- package/engine/dictionary-rows.mjs +14 -2
- package/engine/encoding-language.mjs +47 -20
- package/engine/encoding-lexicon.mjs +25 -3
- package/engine/heading-attributes.mjs +2 -5
- package/engine/language-names.mjs +67 -8
- package/engine/language-writings.mjs +34 -1
- package/engine/markdown-blocks.mjs +58 -0
- package/engine/note-vocabulary.mjs +22 -2
- package/package.json +1 -1
- package/types/engine/content-index.d.mts +28 -2
- package/types/engine/content-session.d.mts +1 -0
- package/types/engine/content-timings.d.mts +7 -0
- package/types/engine/encoding-lexicon.d.mts +6 -35
- package/types/engine/language-names.d.mts +3 -0
- package/types/engine/language-writings.d.mts +3 -1
- package/types/engine/markdown-blocks.d.mts +8 -0
|
@@ -12,6 +12,7 @@
|
|
|
12
12
|
*/
|
|
13
13
|
|
|
14
14
|
import MarkdownIt from "markdown-it";
|
|
15
|
+
import { blockTokens } from "./markdown-blocks.mjs";
|
|
15
16
|
import { parseExtensionAttributes, refusedAttributes } from "./extension-attributes.mjs";
|
|
16
17
|
import { slugify } from "./content-slug.mjs";
|
|
17
18
|
|
|
@@ -48,7 +49,6 @@ export const ALERT_TYPES = Object.freeze({
|
|
|
48
49
|
),
|
|
49
50
|
},
|
|
50
51
|
});
|
|
51
|
-
const parser = new MarkdownIt({ html: true });
|
|
52
52
|
const titleParser = new MarkdownIt({ html: false });
|
|
53
53
|
const marker = /^ {0,3}>[ \t]?\[!([^\]\n]+)\](.*)$/;
|
|
54
54
|
/** Read alerts outside code examples, retaining inclusive source line ranges.
|
|
@@ -60,7 +60,7 @@ export function scanAlerts(source) {
|
|
|
60
60
|
blocks = [],
|
|
61
61
|
errors = [];
|
|
62
62
|
let depth = 0;
|
|
63
|
-
for (const token of
|
|
63
|
+
for (const token of blockTokens(text)) {
|
|
64
64
|
if (token.type === "blockquote_close") {
|
|
65
65
|
depth--;
|
|
66
66
|
continue;
|
|
@@ -52,6 +52,7 @@
|
|
|
52
52
|
import crypto from "node:crypto";
|
|
53
53
|
import { slugify } from "./content-slug.mjs";
|
|
54
54
|
import MarkdownIt from "markdown-it";
|
|
55
|
+
import { blockTokens } from "./markdown-blocks.mjs";
|
|
55
56
|
import { parseExtensionAttributes, refusedAttributes } from "./extension-attributes.mjs";
|
|
56
57
|
import { scanVerse, renderVerse } from "./content-verse.mjs";
|
|
57
58
|
import { renderScroll, scanScrolls } from "./content-scroll.mjs";
|
|
@@ -63,7 +64,6 @@ import { WITHHELD_CLASS, parseHeadingLine, withheldSections } from "./heading-at
|
|
|
63
64
|
* and a stray `<section>` would close the block early.
|
|
64
65
|
*/
|
|
65
66
|
const titleParser = new MarkdownIt({ html: false });
|
|
66
|
-
const structureParser = new MarkdownIt({ html: true });
|
|
67
67
|
|
|
68
68
|
/** The blocks an author may open, with the heading each takes by default. */
|
|
69
69
|
export const BLOCK_NAMES = Object.freeze({
|
|
@@ -108,7 +108,7 @@ export function scanBlocks(source) {
|
|
|
108
108
|
// A bare fence can belong to a quote through lazy continuation.
|
|
109
109
|
// Parser ranges decide whether it can delimit a container.
|
|
110
110
|
const literalLines = new Set();
|
|
111
|
-
for (const token of
|
|
111
|
+
for (const token of blockTokens(source)) {
|
|
112
112
|
if (
|
|
113
113
|
!token.map ||
|
|
114
114
|
!["blockquote_open", "fence", "code_block", "html_block"].includes(token.type)
|
package/engine/content-index.mjs
CHANGED
|
@@ -1087,6 +1087,8 @@ export function resolveCultures(
|
|
|
1087
1087
|
* @param {Map<string, {raw: string}>} [options.sources] - Selected captured source strings.
|
|
1088
1088
|
* @param {import("./content-session.mjs").ContentSyntax} [options.syntax] - Session scanner projections.
|
|
1089
1089
|
* @param {Iterable<string>} [options.nameTypes] - Additional bare name kinds from configured type declarations.
|
|
1090
|
+
* @param {ReturnType<typeof import("./content-timings.mjs").createContentTimings>} [options.timings] -
|
|
1091
|
+
* Invocation stage accounting, which records each phase of the derivation.
|
|
1090
1092
|
* @returns {Array<Record<string, any>>} The records, in a total order that does
|
|
1091
1093
|
* not depend on directory-read order.
|
|
1092
1094
|
*/
|
|
@@ -1109,8 +1111,16 @@ export function collectContentIndex(
|
|
|
1109
1111
|
sources,
|
|
1110
1112
|
syntax,
|
|
1111
1113
|
nameTypes,
|
|
1114
|
+
timings,
|
|
1112
1115
|
},
|
|
1113
1116
|
) {
|
|
1117
|
+
// Each phase runs until the next begins; the caller's stage closes the last.
|
|
1118
|
+
let phaseEnd;
|
|
1119
|
+
const phase = (name) => {
|
|
1120
|
+
phaseEnd?.();
|
|
1121
|
+
phaseEnd = timings?.start(name);
|
|
1122
|
+
};
|
|
1123
|
+
phase("index read");
|
|
1114
1124
|
const records = [];
|
|
1115
1125
|
// Passed through rather than defaulted away: an absent scope is the
|
|
1116
1126
|
// caller's omission, and `walkMarkdownTree` says so.
|
|
@@ -1175,6 +1185,7 @@ export function collectContentIndex(
|
|
|
1175
1185
|
);
|
|
1176
1186
|
|
|
1177
1187
|
// Each recorded note with its body, for the words its prose cites.
|
|
1188
|
+
phase("index notes");
|
|
1178
1189
|
const bodies = [];
|
|
1179
1190
|
for (const { frontmatter, body, bodyLine, bodyColumn, raw, absPath } of parsedNotes) {
|
|
1180
1191
|
// Before the index derives anything onto the frontmatter, so a later
|
|
@@ -1251,16 +1262,19 @@ export function collectContentIndex(
|
|
|
1251
1262
|
|
|
1252
1263
|
// The words each note's prose cites, read once every language's words
|
|
1253
1264
|
// are recorded; the index keeps the citing Addresses and no text.
|
|
1265
|
+
phase("index citations");
|
|
1254
1266
|
attachCitations(records, bodies);
|
|
1255
1267
|
|
|
1256
1268
|
// A run marked with a language alone takes its language's script, which
|
|
1257
1269
|
// is read once every language note is recorded.
|
|
1270
|
+
phase("index encoding runs");
|
|
1258
1271
|
fillEncodingRuns(records, { languages, contentPackage, rootDir });
|
|
1259
1272
|
|
|
1260
1273
|
// Every note record exists and every Address in it is a tuple, so a parent
|
|
1261
1274
|
// is found by its canonical address. Before the sort, which reads nothing
|
|
1262
1275
|
// the pass writes. Calendars first: they read the culture a note states,
|
|
1263
1276
|
// which the culture pass then replaces with the one it resolves.
|
|
1277
|
+
phase("index calendars and cultures");
|
|
1264
1278
|
resolveNoteCalendars(records, {
|
|
1265
1279
|
contentPackage,
|
|
1266
1280
|
defaultCalendar,
|
|
@@ -1298,6 +1312,7 @@ export function collectContentIndex(
|
|
|
1298
1312
|
// projection as the surfaces, without making the index emitter async.
|
|
1299
1313
|
// Public notes resolve providers in their audience, including lineage
|
|
1300
1314
|
// through hidden languages, just as their build does.
|
|
1315
|
+
phase("index lexical sections");
|
|
1301
1316
|
const visible = records.filter((record) => !isGmNote(record));
|
|
1302
1317
|
const publicSections = lexicalSectionsFor(visible, records);
|
|
1303
1318
|
const allSections = lexicalSectionsFor(records);
|
|
@@ -1319,6 +1334,7 @@ export function collectContentIndex(
|
|
|
1319
1334
|
// about the content, and the artifact is meant to be byte-identical across
|
|
1320
1335
|
// two runs over an unchanged tree. Collected above, ahead of the notes —
|
|
1321
1336
|
// see `resolveRole` — rather than walked a second time here.
|
|
1337
|
+
phase("index order");
|
|
1322
1338
|
for (const record of assetRecords) {
|
|
1323
1339
|
records.push(/** @type {Record<string, any>} */ (sortKeysDeep(record)));
|
|
1324
1340
|
}
|
|
@@ -1344,9 +1360,11 @@ export function collectContentIndex(
|
|
|
1344
1360
|
) ||
|
|
1345
1361
|
String(a.id ?? "").localeCompare(String(b.id ?? ""), "en"),
|
|
1346
1362
|
);
|
|
1347
|
-
|
|
1363
|
+
const decoded = records.map((record) =>
|
|
1348
1364
|
decodeIndexAddresses(record, addressContext ?? { package: contentPackage }),
|
|
1349
1365
|
);
|
|
1366
|
+
phaseEnd?.();
|
|
1367
|
+
return decoded;
|
|
1350
1368
|
}
|
|
1351
1369
|
|
|
1352
1370
|
/**
|
|
@@ -1465,6 +1483,8 @@ function dependencyAddresses(config) {
|
|
|
1465
1483
|
* @param {import("./content-session.mjs").ContentSyntax} [opts.syntax] - Session scanner projections.
|
|
1466
1484
|
* @param {Iterable<string>} [opts.nameTypes] - Bare name kinds, defaulting to configured item types.
|
|
1467
1485
|
* @param {Map<string, object>} [opts.foreignIndex] - Captured dependency address view.
|
|
1486
|
+
* @param {ReturnType<typeof import("./content-timings.mjs").createContentTimings>} [opts.timings] -
|
|
1487
|
+
* Invocation stage accounting.
|
|
1468
1488
|
* @returns {object[]} One record per note, plus one per documentation entry.
|
|
1469
1489
|
*/
|
|
1470
1490
|
export function indexRecordsFor({
|
|
@@ -1478,6 +1498,7 @@ export function indexRecordsFor({
|
|
|
1478
1498
|
syntax,
|
|
1479
1499
|
nameTypes,
|
|
1480
1500
|
foreignIndex,
|
|
1501
|
+
timings,
|
|
1481
1502
|
} = {}) {
|
|
1482
1503
|
const resolved = config ?? loadPackConfig();
|
|
1483
1504
|
const tree = contentBase ?? resolved.paths.content;
|
|
@@ -1512,6 +1533,7 @@ export function indexRecordsFor({
|
|
|
1512
1533
|
sources,
|
|
1513
1534
|
syntax,
|
|
1514
1535
|
nameTypes: nameTypes ?? resolved.itemTypes,
|
|
1536
|
+
timings,
|
|
1515
1537
|
});
|
|
1516
1538
|
}
|
|
1517
1539
|
|
|
@@ -41,6 +41,7 @@
|
|
|
41
41
|
*/
|
|
42
42
|
|
|
43
43
|
import MarkdownIt from "markdown-it";
|
|
44
|
+
import { blockTokens } from "./markdown-blocks.mjs";
|
|
44
45
|
|
|
45
46
|
import { scanBlocks, BLOCK_CONTAINERS } from "./content-blocks.mjs";
|
|
46
47
|
import { scanVerse, hasColumns, verseLine, verseGroups, STANZA_BREAK } from "./content-verse.mjs";
|
|
@@ -737,9 +738,6 @@ function codeName(point) {
|
|
|
737
738
|
return `U+${point.toString(16).toUpperCase().padStart(4, "0")}`;
|
|
738
739
|
}
|
|
739
740
|
|
|
740
|
-
/** A parser for the block structure of a div's body. */
|
|
741
|
-
const blockParser = new MarkdownIt({ html: true });
|
|
742
|
-
|
|
743
741
|
/** What a block token is called in a finding. */
|
|
744
742
|
const BLOCK_WHAT = Object.freeze({
|
|
745
743
|
heading_open: "a heading",
|
|
@@ -1485,7 +1483,7 @@ function analyseRuns(source, ctx, syntax) {
|
|
|
1485
1483
|
{ line: first + line + 1, column: /^ */.exec(body[line])[0].length + 1 },
|
|
1486
1484
|
`a div marked \`${marking}\` holds paragraphs and line breaks only, not ${what}`,
|
|
1487
1485
|
);
|
|
1488
|
-
for (const token of
|
|
1486
|
+
for (const token of blockTokens(body.join("\n"))) {
|
|
1489
1487
|
if (token.level !== 0 || !token.map) continue;
|
|
1490
1488
|
const [top, past] = token.map;
|
|
1491
1489
|
if (token.type === "paragraph_open") {
|
|
@@ -1535,7 +1533,14 @@ function analyseRuns(source, ctx, syntax) {
|
|
|
1535
1533
|
const boundsOf = (detail) =>
|
|
1536
1534
|
detail.reference ??
|
|
1537
1535
|
(detail.run.kind === "span" ? [detail.run.start + 1, detail.contentEnd] : detail.body);
|
|
1536
|
+
// Each run's parent and owner are read many times while ownership is
|
|
1537
|
+
// assigned below, so each is found once.
|
|
1538
|
+
const parents = new Map();
|
|
1538
1539
|
const parentOf = (detail) => {
|
|
1540
|
+
if (!parents.has(detail)) parents.set(detail, findParent(detail));
|
|
1541
|
+
return parents.get(detail);
|
|
1542
|
+
};
|
|
1543
|
+
const findParent = (detail) => {
|
|
1539
1544
|
if (detail.run.kind !== "span" || Object.keys(detail.keys).length) return undefined;
|
|
1540
1545
|
return details
|
|
1541
1546
|
.filter(
|
|
@@ -1547,9 +1552,13 @@ function analyseRuns(source, ctx, syntax) {
|
|
|
1547
1552
|
)
|
|
1548
1553
|
.sort((a, b) => boundsOf(a)[1] - boundsOf(a)[0] - (boundsOf(b)[1] - boundsOf(b)[0]))[0];
|
|
1549
1554
|
};
|
|
1555
|
+
const owners = new Map();
|
|
1550
1556
|
const ownerOf = (detail) => {
|
|
1551
|
-
|
|
1552
|
-
|
|
1557
|
+
if (!owners.has(detail)) {
|
|
1558
|
+
const parent = parentOf(detail);
|
|
1559
|
+
owners.set(detail, parent ? ownerOf(parent) : detail);
|
|
1560
|
+
}
|
|
1561
|
+
return owners.get(detail);
|
|
1553
1562
|
};
|
|
1554
1563
|
const wholeVerseReading = (detail) => {
|
|
1555
1564
|
if (detail.run.kind === "verse") return true;
|
|
@@ -42,7 +42,7 @@
|
|
|
42
42
|
* @module
|
|
43
43
|
*/
|
|
44
44
|
|
|
45
|
-
import
|
|
45
|
+
import { blockTokens } from "./markdown-blocks.mjs";
|
|
46
46
|
|
|
47
47
|
import { parseExtensionAttributes, refusedAttributes } from "./extension-attributes.mjs";
|
|
48
48
|
|
|
@@ -78,8 +78,6 @@ export const BOOK_SCROLL_ROWS = 40;
|
|
|
78
78
|
/** The class every scroll carries. */
|
|
79
79
|
export const SCROLL_CLASS = "scroll";
|
|
80
80
|
|
|
81
|
-
const structureParser = new MarkdownIt({ html: true });
|
|
82
|
-
|
|
83
81
|
/**
|
|
84
82
|
* One `scroll` fence.
|
|
85
83
|
*
|
|
@@ -129,7 +127,7 @@ export function scanScrolls(source) {
|
|
|
129
127
|
if (!/^ *(?:`{3,}|~{3,}) *scroll(?:\s|$)/m.test(text)) return { blocks, errors };
|
|
130
128
|
const lines = text.split("\n");
|
|
131
129
|
const literalLines = new Set();
|
|
132
|
-
for (const token of
|
|
130
|
+
for (const token of blockTokens(text)) {
|
|
133
131
|
if (!token.map || !["blockquote_open", "html_block"].includes(token.type)) continue;
|
|
134
132
|
for (let line = token.map[0]; line < token.map[1]; line++) literalLines.add(line);
|
|
135
133
|
}
|
|
@@ -24,12 +24,15 @@ export function createContentTimings() {
|
|
|
24
24
|
stages.set(name, (stages.get(name) ?? 0) + now - changed);
|
|
25
25
|
changed = now;
|
|
26
26
|
};
|
|
27
|
+
// Ending a stage also ends any stage opened inside it and left open.
|
|
27
28
|
const begin = (name) => {
|
|
28
29
|
account();
|
|
30
|
+
const depth = stack.length;
|
|
29
31
|
stack.push(name);
|
|
30
32
|
return () => {
|
|
33
|
+
if (stack.length <= depth) return;
|
|
31
34
|
account();
|
|
32
|
-
stack.
|
|
35
|
+
stack.length = depth;
|
|
33
36
|
};
|
|
34
37
|
};
|
|
35
38
|
return {
|
|
@@ -51,6 +54,15 @@ export function createContentTimings() {
|
|
|
51
54
|
stop();
|
|
52
55
|
}
|
|
53
56
|
},
|
|
57
|
+
/**
|
|
58
|
+
* Open a stage that runs until the returned function is called, or
|
|
59
|
+
* until the stage it was opened in ends.
|
|
60
|
+
* @param {string} name
|
|
61
|
+
* @returns {() => void}
|
|
62
|
+
*/
|
|
63
|
+
start(name) {
|
|
64
|
+
return begin(name);
|
|
65
|
+
},
|
|
54
66
|
/** @param {string} name */
|
|
55
67
|
hit(name) {
|
|
56
68
|
hits.set(name, (hits.get(name) ?? 0) + 1);
|
package/engine/content-verse.mjs
CHANGED
|
@@ -180,11 +180,11 @@ function columnsMarkdown(block, lines, step) {
|
|
|
180
180
|
}
|
|
181
181
|
|
|
182
182
|
import MarkdownIt from "markdown-it";
|
|
183
|
+
import { blockTokens } from "./markdown-blocks.mjs";
|
|
183
184
|
import { slugify } from "./content-slug.mjs";
|
|
184
185
|
import { spanMarkdownPlugin } from "./content-spans.mjs";
|
|
185
186
|
import { parseExtensionAttributes, refusedAttributes } from "./extension-attributes.mjs";
|
|
186
187
|
|
|
187
|
-
const structureParser = new MarkdownIt({ html: true });
|
|
188
188
|
const lineParser = new MarkdownIt({ html: true }).use(spanMarkdownPlugin);
|
|
189
189
|
|
|
190
190
|
/** The structural first span of a line, with its source ranges and keys. */
|
|
@@ -223,7 +223,7 @@ export function scanVerse(source) {
|
|
|
223
223
|
blocks = [],
|
|
224
224
|
errors = [];
|
|
225
225
|
const literalLines = new Set();
|
|
226
|
-
for (const token of
|
|
226
|
+
for (const token of blockTokens(source)) {
|
|
227
227
|
if (!token.map || !["blockquote_open", "html_block"].includes(token.type)) continue;
|
|
228
228
|
for (let line = token.map[0]; line < token.map[1]; line++) literalLines.add(line);
|
|
229
229
|
}
|
|
@@ -464,6 +464,16 @@ export function dictionaryAndTermRows(records, all = records) {
|
|
|
464
464
|
);
|
|
465
465
|
return hit ? hit.entry[key] : undefined;
|
|
466
466
|
};
|
|
467
|
+
// One piece lookup per language, shared by every word that language states.
|
|
468
|
+
const glossers = new Map();
|
|
469
|
+
const glossesFor = (language) => {
|
|
470
|
+
if (!glossers.has(language))
|
|
471
|
+
glossers.set(
|
|
472
|
+
language,
|
|
473
|
+
pieceGlosses(...lineage(language).map((key) => languages.get(key).data)),
|
|
474
|
+
);
|
|
475
|
+
return glossers.get(language);
|
|
476
|
+
};
|
|
467
477
|
const etymologyOf = ({ word, statedIn, depth }) => {
|
|
468
478
|
const blank = Object.fromEntries(ETYMOLOGY_KEYS.map((key) => [key, null]));
|
|
469
479
|
if (depth > 0)
|
|
@@ -476,13 +486,15 @@ export function dictionaryAndTermRows(records, all = records) {
|
|
|
476
486
|
};
|
|
477
487
|
const source = isMap(word.source) ? word.source : {};
|
|
478
488
|
const built = Array.isArray(word.built) ? word.built.map(String) : [];
|
|
479
|
-
const glossOf = pieceGlosses(...lineage(statedIn).map((key) => languages.get(key).data));
|
|
480
489
|
const out = {
|
|
481
490
|
...blank,
|
|
482
491
|
kind: typeof source.kind === "string" ? source.kind : null,
|
|
483
492
|
language: typeof source.language === "string" ? source.language : null,
|
|
484
493
|
form: typeof source.form === "string" ? source.form : null,
|
|
485
|
-
parts:
|
|
494
|
+
parts:
|
|
495
|
+
built.length ?
|
|
496
|
+
built.map((form) => ({ form, gloss: glossesFor(statedIn)(form) }))
|
|
497
|
+
: null,
|
|
486
498
|
root: typeof word.root === "string" ? word.root : null,
|
|
487
499
|
frame: typeof word.frame === "string" ? word.frame : null,
|
|
488
500
|
};
|
|
@@ -69,7 +69,14 @@ function capitalWriting(enc, word, native, font) {
|
|
|
69
69
|
upper.to + native.slice(lower.to.length)
|
|
70
70
|
: undefined;
|
|
71
71
|
}
|
|
72
|
+
/** Each lexicon's forms, found once; a lexicon is not modified once it is converted with. */
|
|
73
|
+
const formsOf = new WeakMap();
|
|
72
74
|
function entryForms(lexicon) {
|
|
75
|
+
if (!lexicon) return findEntryForms(lexicon);
|
|
76
|
+
if (!formsOf.has(lexicon)) formsOf.set(lexicon, findEntryForms(lexicon));
|
|
77
|
+
return formsOf.get(lexicon);
|
|
78
|
+
}
|
|
79
|
+
function findEntryForms(lexicon) {
|
|
73
80
|
const data = lexicon?.data ?? {};
|
|
74
81
|
const derived = new Map();
|
|
75
82
|
for (const entry of derivedForms(data)) {
|
|
@@ -122,6 +129,44 @@ function entryForms(lexicon) {
|
|
|
122
129
|
];
|
|
123
130
|
}
|
|
124
131
|
|
|
132
|
+
/** Each lexicon's phrases of more than one word, with the patterns that find them. */
|
|
133
|
+
const phrasesFound = new WeakMap();
|
|
134
|
+
function phrasesOf(lexicon) {
|
|
135
|
+
if (!phrasesFound.has(lexicon))
|
|
136
|
+
phrasesFound.set(
|
|
137
|
+
lexicon,
|
|
138
|
+
entryForms(lexicon)
|
|
139
|
+
.map((entry) => entry.form)
|
|
140
|
+
.filter((form) => [...form.matchAll(WORD)].length > 1)
|
|
141
|
+
.map((form) => ({
|
|
142
|
+
form,
|
|
143
|
+
spaces: form.match(/[\p{Zs}\u0009-\u000D]+/gu) ?? [],
|
|
144
|
+
pattern: new RegExp(
|
|
145
|
+
form
|
|
146
|
+
.split(/[\p{Zs}\u0009-\u000D]+/u)
|
|
147
|
+
.map((part) => part.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"))
|
|
148
|
+
.join("[\\p{Zs}\\u0009-\\u000D]+"),
|
|
149
|
+
"iuy",
|
|
150
|
+
),
|
|
151
|
+
})),
|
|
152
|
+
);
|
|
153
|
+
return phrasesFound.get(lexicon);
|
|
154
|
+
}
|
|
155
|
+
|
|
156
|
+
/** Each lexicon's words' forms and variants, as written. */
|
|
157
|
+
const statedFound = new WeakMap();
|
|
158
|
+
function statedForms(lexicon) {
|
|
159
|
+
if (!statedFound.has(lexicon)) {
|
|
160
|
+
const forms = new Set();
|
|
161
|
+
for (const entry of list(lexicon.data.words)) {
|
|
162
|
+
forms.add(entry.form);
|
|
163
|
+
for (const variant of list(entry.variants)) forms.add(variant);
|
|
164
|
+
}
|
|
165
|
+
statedFound.set(lexicon, forms);
|
|
166
|
+
}
|
|
167
|
+
return statedFound.get(lexicon);
|
|
168
|
+
}
|
|
169
|
+
|
|
125
170
|
function literalEnd(text, at) {
|
|
126
171
|
let cursor = at + OPEN.length;
|
|
127
172
|
while (cursor < text.length) {
|
|
@@ -164,23 +209,7 @@ export function encodeWithLanguage(enc, text, options = {}) {
|
|
|
164
209
|
}
|
|
165
210
|
const writing =
|
|
166
211
|
options.lexicon ? writingsOf(options.lexicon.data, enc.name, options.font) : undefined;
|
|
167
|
-
const phrases =
|
|
168
|
-
options.lexicon ?
|
|
169
|
-
entryForms(options.lexicon)
|
|
170
|
-
.map((entry) => entry.form)
|
|
171
|
-
.filter((form) => [...form.matchAll(WORD)].length > 1)
|
|
172
|
-
.map((form) => ({
|
|
173
|
-
form,
|
|
174
|
-
spaces: form.match(/[\p{Zs}\u0009-\u000D]+/gu) ?? [],
|
|
175
|
-
pattern: new RegExp(
|
|
176
|
-
form
|
|
177
|
-
.split(/[\p{Zs}\u0009-\u000D]+/u)
|
|
178
|
-
.map((part) => part.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"))
|
|
179
|
-
.join("[\\p{Zs}\\u0009-\\u000D]+"),
|
|
180
|
-
"iuy",
|
|
181
|
-
),
|
|
182
|
-
}))
|
|
183
|
-
: [];
|
|
212
|
+
const phrases = options.lexicon ? phrasesOf(options.lexicon) : [];
|
|
184
213
|
let at = 0;
|
|
185
214
|
const appendRows = (from, to) => {
|
|
186
215
|
let cursor = from;
|
|
@@ -267,9 +296,7 @@ export function encodeWithLanguage(enc, text, options = {}) {
|
|
|
267
296
|
} else if (own && "text" in own) {
|
|
268
297
|
appendRows(at, start);
|
|
269
298
|
let native = own.text;
|
|
270
|
-
const exact =
|
|
271
|
-
(entry) => entry.form === lookup || list(entry.variants).includes(lookup),
|
|
272
|
-
);
|
|
299
|
+
const exact = statedForms(options.lexicon).has(lookup);
|
|
273
300
|
if (!exact && match[0] !== match[0].toLowerCase()) {
|
|
274
301
|
const capital =
|
|
275
302
|
match[0] === initialCapital(match[0].toLowerCase()) ?
|
|
@@ -22,6 +22,7 @@ import { isLanguageRecord, lineageChain, noteAddressOf } from "./word-rows.mjs";
|
|
|
22
22
|
import { derivedForms, foldWord } from "./word-forms.mjs";
|
|
23
23
|
import { WRITTEN_ENDINGS, RESERVED_TONGUE } from "./language-terms.mjs";
|
|
24
24
|
import { LANGUAGE_SUBTYPE, glossText } from "./word-terms.mjs";
|
|
25
|
+
import { ENGLISH_GROUP } from "./language-names.mjs";
|
|
25
26
|
import { writingsOf } from "./language-writings.mjs";
|
|
26
27
|
import { encodeText, validateSource, EncodingError } from "./encoding.mjs";
|
|
27
28
|
import { encodeWithLanguage } from "./encoding-language.mjs";
|
|
@@ -389,9 +390,21 @@ export function nativeRows(records) {
|
|
|
389
390
|
return rows;
|
|
390
391
|
}
|
|
391
392
|
|
|
392
|
-
/**
|
|
393
|
+
/** Each lexicon's projection, made once and shared by every conversion that reads it. */
|
|
394
|
+
const serialized = new WeakMap();
|
|
395
|
+
|
|
396
|
+
/**
|
|
397
|
+
* Project only conversion-readable language data to a JSON-safe object. The
|
|
398
|
+
* projection of one lexicon is made once and shared, so it must not be modified.
|
|
399
|
+
*/
|
|
393
400
|
export function serializeLexicon(lexicon) {
|
|
394
401
|
if (!lexicon) return undefined;
|
|
402
|
+
if (!serialized.has(lexicon)) serialized.set(lexicon, projectLexicon(lexicon));
|
|
403
|
+
return serialized.get(lexicon);
|
|
404
|
+
}
|
|
405
|
+
|
|
406
|
+
/** The conversion-readable data of a lexicon, copied. */
|
|
407
|
+
function projectLexicon(lexicon) {
|
|
395
408
|
const select = (value, keys) =>
|
|
396
409
|
Object.fromEntries(
|
|
397
410
|
keys
|
|
@@ -630,7 +643,10 @@ export function checkEncodingLexicon(note, ctx) {
|
|
|
630
643
|
...new Map(
|
|
631
644
|
[...(ctx.encodings?.values() ?? [])].map((encoding) => [encoding.name, encoding]),
|
|
632
645
|
).values(),
|
|
633
|
-
]
|
|
646
|
+
]
|
|
647
|
+
.filter((encoding) => aliases.has(encoding.language))
|
|
648
|
+
// A codes-only script accepts no letter, so the roman forms are not written in it.
|
|
649
|
+
.filter((encoding) => [...encoding.accepted].some((char) => /\p{L}/u.test(char)));
|
|
634
650
|
if (!encodings.length) return [];
|
|
635
651
|
const forms = [],
|
|
636
652
|
data = note.fm.data ?? {};
|
|
@@ -641,6 +657,7 @@ export function checkEncodingLexicon(note, ctx) {
|
|
|
641
657
|
add(word?.form, ["words", i, "form"]);
|
|
642
658
|
listOf(word?.variants).forEach((form, j) => add(form, ["words", i, "variants", j]));
|
|
643
659
|
});
|
|
660
|
+
// `en` holds English names, which no encoding of the tongue sees.
|
|
644
661
|
const names = (value, keys) => {
|
|
645
662
|
if (Array.isArray(value))
|
|
646
663
|
value.forEach((item, i) =>
|
|
@@ -648,7 +665,12 @@ export function checkEncodingLexicon(note, ctx) {
|
|
|
648
665
|
);
|
|
649
666
|
else if (isMap(value))
|
|
650
667
|
for (const [key, item] of Object.entries(value))
|
|
651
|
-
if (
|
|
668
|
+
if (
|
|
669
|
+
key !== "pattern" &&
|
|
670
|
+
key !== "metadata" &&
|
|
671
|
+
!(keys.length === 1 && key === ENGLISH_GROUP)
|
|
672
|
+
)
|
|
673
|
+
names(item, [...keys, key]);
|
|
652
674
|
};
|
|
653
675
|
names(data.names, ["names"]);
|
|
654
676
|
listOf(data.elements).forEach((element, i) => {
|
|
@@ -34,15 +34,12 @@
|
|
|
34
34
|
*/
|
|
35
35
|
|
|
36
36
|
import { parseExtensionAttributes, refusedAttributes } from "./extension-attributes.mjs";
|
|
37
|
-
import
|
|
38
|
-
|
|
39
|
-
/** Literal block ranges use the renderer's Markdown grammar. */
|
|
40
|
-
const structureParser = new MarkdownIt({ html: true });
|
|
37
|
+
import { blockTokens } from "./markdown-blocks.mjs";
|
|
41
38
|
|
|
42
39
|
/** Body lines occupied by code or raw HTML, including comment examples. */
|
|
43
40
|
function literalHeadingLines(source) {
|
|
44
41
|
const lines = new Set();
|
|
45
|
-
for (const token of
|
|
42
|
+
for (const token of blockTokens(source)) {
|
|
46
43
|
if (!token.map || !["fence", "code_block", "html_block"].includes(token.type)) continue;
|
|
47
44
|
for (let line = token.map[0]; line < token.map[1]; line++) lines.add(line);
|
|
48
45
|
}
|
|
@@ -33,6 +33,10 @@
|
|
|
33
33
|
* person's gender's list, or from every given list together where that gender
|
|
34
34
|
* has none; `{given.<gender>}` draws from that gender's list whatever the
|
|
35
35
|
* person's gender, as a patronymic draws a father's name.
|
|
36
|
+
* - `en` — English names, in groups of lists: `en.epithet` is a list a pattern
|
|
37
|
+
* draws from as `{en.epithet}`, as `"{given} the {en.epithet}"` does. English
|
|
38
|
+
* names are never encoded in the tongue's script or checked against its
|
|
39
|
+
* spelling.
|
|
36
40
|
* - every other key — a list of names the pattern draws from by that key:
|
|
37
41
|
* `clan`, `family`, `place`. A list name is letters, digits and underscores,
|
|
38
42
|
* beginning with a letter.
|
|
@@ -55,13 +59,16 @@ import { nearest } from "./near-miss.mjs";
|
|
|
55
59
|
/** The `skill` subType whose notes may carry name lists. */
|
|
56
60
|
export const NAMES_SUBTYPE = "language";
|
|
57
61
|
|
|
62
|
+
/** The key whose groups of lists hold English names. */
|
|
63
|
+
export const ENGLISH_GROUP = "en";
|
|
64
|
+
|
|
58
65
|
/** The placeholder that draws from the given-name list for a gender. */
|
|
59
66
|
export const GIVEN_LIST = "given";
|
|
60
67
|
|
|
61
68
|
/** What a list name may be spelled with. */
|
|
62
69
|
const LIST_NAME = /^[A-Za-z][A-Za-z0-9_]*$/;
|
|
63
70
|
|
|
64
|
-
/** A placeholder's text: a list name, and for `given`
|
|
71
|
+
/** A placeholder's text: a list name, and after a dot a gender for `given` or a list for `en`. */
|
|
65
72
|
const PLACEHOLDER = /^([A-Za-z][A-Za-z0-9_]*)(?:\.([A-Za-z][A-Za-z0-9_]*))?$/;
|
|
66
73
|
|
|
67
74
|
/**
|
|
@@ -80,11 +87,12 @@ export function requiredGivenGenders() {
|
|
|
80
87
|
* One piece of a parsed pattern: literal text, or a placeholder naming a list
|
|
81
88
|
* and, for `given`, the gender whose list it draws from.
|
|
82
89
|
*
|
|
83
|
-
* @typedef {{literal: string} | {list: string, gender?: string}} PatternPart
|
|
90
|
+
* @typedef {{literal: string} | {list: string, gender?: string, sub?: string}} PatternPart
|
|
84
91
|
*/
|
|
85
92
|
|
|
86
93
|
/** A placeholder as written, without its braces. */
|
|
87
94
|
function placeholderText(part) {
|
|
95
|
+
if (part.sub !== undefined) return `${part.list}.${part.sub}`;
|
|
88
96
|
return part.gender ? `${part.list}.${part.gender}` : part.list;
|
|
89
97
|
}
|
|
90
98
|
|
|
@@ -126,17 +134,22 @@ export function parsePattern(pattern) {
|
|
|
126
134
|
`has the placeholder \`{${text}}\`; a placeholder names a list in letters, ` +
|
|
127
135
|
"digits and underscores, beginning with a letter",
|
|
128
136
|
);
|
|
129
|
-
else if (match[
|
|
137
|
+
else if (match[1] === ENGLISH_GROUP && match[2] === undefined)
|
|
138
|
+
problems.push(
|
|
139
|
+
`has the placeholder \`{${text}}\`; \`en\` holds groups of English names, ` +
|
|
140
|
+
"so a placeholder names one as `{en.<list>}`",
|
|
141
|
+
);
|
|
142
|
+
else if (match[2] !== undefined && match[1] !== GIVEN_LIST && match[1] !== ENGLISH_GROUP)
|
|
130
143
|
problems.push(
|
|
131
144
|
`has the placeholder \`{${text}}\`; only \`given\` takes a gender after a dot, ` +
|
|
132
|
-
"as `{given.male}` does",
|
|
145
|
+
"as `{given.male}` does, and `en` a list, as `{en.epithet}` does",
|
|
133
146
|
);
|
|
134
147
|
else {
|
|
135
148
|
if (literal) parts.push({ literal });
|
|
136
149
|
literal = "";
|
|
137
|
-
parts.push(
|
|
138
|
-
|
|
139
|
-
);
|
|
150
|
+
if (match[2] === undefined) parts.push({ list: match[1] });
|
|
151
|
+
else if (match[1] === ENGLISH_GROUP) parts.push({ list: match[1], sub: match[2] });
|
|
152
|
+
else parts.push({ list: match[1], gender: match[2] });
|
|
140
153
|
}
|
|
141
154
|
i = close + 1;
|
|
142
155
|
}
|
|
@@ -263,7 +276,30 @@ export function namesProblems(names, lookVocabulary) {
|
|
|
263
276
|
});
|
|
264
277
|
};
|
|
265
278
|
|
|
266
|
-
const lists = Object.keys(names).filter((key) => key !== "pattern");
|
|
279
|
+
const lists = Object.keys(names).filter((key) => key !== "pattern" && key !== ENGLISH_GROUP);
|
|
280
|
+
const english = names[ENGLISH_GROUP];
|
|
281
|
+
const englishLists = [];
|
|
282
|
+
if (english !== undefined && english !== null) {
|
|
283
|
+
if (!isPlainMap(english))
|
|
284
|
+
shape(
|
|
285
|
+
[ENGLISH_GROUP],
|
|
286
|
+
`\`${label([ENGLISH_GROUP])}\` should be a map of names to lists of English names`,
|
|
287
|
+
);
|
|
288
|
+
else
|
|
289
|
+
for (const key of Object.keys(english)) {
|
|
290
|
+
if (!LIST_NAME.test(key)) {
|
|
291
|
+
problem(
|
|
292
|
+
[ENGLISH_GROUP, key],
|
|
293
|
+
`"${key}" is not a list name; a list of \`${label([ENGLISH_GROUP])}\` is ` +
|
|
294
|
+
"named in letters, digits and underscores, beginning with a letter",
|
|
295
|
+
true,
|
|
296
|
+
);
|
|
297
|
+
continue;
|
|
298
|
+
}
|
|
299
|
+
englishLists.push(key);
|
|
300
|
+
checkList([ENGLISH_GROUP, key]);
|
|
301
|
+
}
|
|
302
|
+
}
|
|
267
303
|
for (const key of lists) {
|
|
268
304
|
if (key === GIVEN_LIST) continue;
|
|
269
305
|
if (!LIST_NAME.test(key)) {
|
|
@@ -376,6 +412,19 @@ export function namesProblems(names, lookVocabulary) {
|
|
|
376
412
|
if (parsed.problems.length) unreadable = true;
|
|
377
413
|
for (const part of parsed.parts) {
|
|
378
414
|
if (!("list" in part)) continue;
|
|
415
|
+
if (part.sub !== undefined) {
|
|
416
|
+
named.add(placeholderText(part));
|
|
417
|
+
if (englishLists.includes(part.sub)) continue;
|
|
418
|
+
const guess = nearest(part.sub, englishLists);
|
|
419
|
+
problem(
|
|
420
|
+
path,
|
|
421
|
+
`\`{${placeholderText(part)}}\` in \`${label(path)}\` names no list of ` +
|
|
422
|
+
`\`${label([ENGLISH_GROUP])}\`` +
|
|
423
|
+
(englishLists.length ? `; the lists are ${englishLists.join(", ")}` : "") +
|
|
424
|
+
(guess ? `. Did you mean "{en.${guess}}"?` : ""),
|
|
425
|
+
);
|
|
426
|
+
continue;
|
|
427
|
+
}
|
|
379
428
|
named.add(part.list);
|
|
380
429
|
if (part.gender !== undefined) {
|
|
381
430
|
if (givenMap[part.gender] === undefined || givenMap[part.gender] === null)
|
|
@@ -397,6 +446,15 @@ export function namesProblems(names, lookVocabulary) {
|
|
|
397
446
|
}
|
|
398
447
|
}
|
|
399
448
|
if (unreadable) return problems;
|
|
449
|
+
for (const list of englishLists) {
|
|
450
|
+
if (named.has(`${ENGLISH_GROUP}.${list}`)) continue;
|
|
451
|
+
problem(
|
|
452
|
+
[ENGLISH_GROUP, list],
|
|
453
|
+
`\`${label([ENGLISH_GROUP, list])}\` is never drawn from: no pattern of ` +
|
|
454
|
+
`\`${label(["pattern"])}\` has \`{${ENGLISH_GROUP}.${list}}\``,
|
|
455
|
+
true,
|
|
456
|
+
);
|
|
457
|
+
}
|
|
400
458
|
for (const list of lists) {
|
|
401
459
|
if (named.has(list)) continue;
|
|
402
460
|
problem(
|
|
@@ -490,6 +548,7 @@ export function givenNames(names, gender) {
|
|
|
490
548
|
|
|
491
549
|
/** The names one placeholder draws from, for a person of `gender`. */
|
|
492
550
|
function listFor(names, part, gender) {
|
|
551
|
+
if (part.sub !== undefined) return names[part.list][part.sub];
|
|
493
552
|
if (part.list !== GIVEN_LIST) return names[part.list];
|
|
494
553
|
if (part.gender !== undefined) return names[GIVEN_LIST][part.gender];
|
|
495
554
|
return givenNames(names, gender);
|