@heroiclands/package-build 25.1.1 → 25.1.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -12,6 +12,7 @@
12
12
  */
13
13
 
14
14
  import MarkdownIt from "markdown-it";
15
+ import { blockTokens } from "./markdown-blocks.mjs";
15
16
  import { parseExtensionAttributes, refusedAttributes } from "./extension-attributes.mjs";
16
17
  import { slugify } from "./content-slug.mjs";
17
18
 
@@ -48,7 +49,6 @@ export const ALERT_TYPES = Object.freeze({
48
49
  ),
49
50
  },
50
51
  });
51
- const parser = new MarkdownIt({ html: true });
52
52
  const titleParser = new MarkdownIt({ html: false });
53
53
  const marker = /^ {0,3}>[ \t]?\[!([^\]\n]+)\](.*)$/;
54
54
  /** Read alerts outside code examples, retaining inclusive source line ranges.
@@ -60,7 +60,7 @@ export function scanAlerts(source) {
60
60
  blocks = [],
61
61
  errors = [];
62
62
  let depth = 0;
63
- for (const token of parser.parse(text, {})) {
63
+ for (const token of blockTokens(text)) {
64
64
  if (token.type === "blockquote_close") {
65
65
  depth--;
66
66
  continue;
@@ -52,6 +52,7 @@
52
52
  import crypto from "node:crypto";
53
53
  import { slugify } from "./content-slug.mjs";
54
54
  import MarkdownIt from "markdown-it";
55
+ import { blockTokens } from "./markdown-blocks.mjs";
55
56
  import { parseExtensionAttributes, refusedAttributes } from "./extension-attributes.mjs";
56
57
  import { scanVerse, renderVerse } from "./content-verse.mjs";
57
58
  import { renderScroll, scanScrolls } from "./content-scroll.mjs";
@@ -63,7 +64,6 @@ import { WITHHELD_CLASS, parseHeadingLine, withheldSections } from "./heading-at
63
64
  * and a stray `<section>` would close the block early.
64
65
  */
65
66
  const titleParser = new MarkdownIt({ html: false });
66
- const structureParser = new MarkdownIt({ html: true });
67
67
 
68
68
  /** The blocks an author may open, with the heading each takes by default. */
69
69
  export const BLOCK_NAMES = Object.freeze({
@@ -108,7 +108,7 @@ export function scanBlocks(source) {
108
108
  // A bare fence can belong to a quote through lazy continuation.
109
109
  // Parser ranges decide whether it can delimit a container.
110
110
  const literalLines = new Set();
111
- for (const token of structureParser.parse(String(source ?? ""), {})) {
111
+ for (const token of blockTokens(source)) {
112
112
  if (
113
113
  !token.map ||
114
114
  !["blockquote_open", "fence", "code_block", "html_block"].includes(token.type)
@@ -1087,6 +1087,8 @@ export function resolveCultures(
1087
1087
  * @param {Map<string, {raw: string}>} [options.sources] - Selected captured source strings.
1088
1088
  * @param {import("./content-session.mjs").ContentSyntax} [options.syntax] - Session scanner projections.
1089
1089
  * @param {Iterable<string>} [options.nameTypes] - Additional bare name kinds from configured type declarations.
1090
+ * @param {ReturnType<typeof import("./content-timings.mjs").createContentTimings>} [options.timings] -
1091
+ * Invocation stage accounting, which records each phase of the derivation.
1090
1092
  * @returns {Array<Record<string, any>>} The records, in a total order that does
1091
1093
  * not depend on directory-read order.
1092
1094
  */
@@ -1109,8 +1111,16 @@ export function collectContentIndex(
1109
1111
  sources,
1110
1112
  syntax,
1111
1113
  nameTypes,
1114
+ timings,
1112
1115
  },
1113
1116
  ) {
1117
+ // Each phase runs until the next begins; the caller's stage closes the last.
1118
+ let phaseEnd;
1119
+ const phase = (name) => {
1120
+ phaseEnd?.();
1121
+ phaseEnd = timings?.start(name);
1122
+ };
1123
+ phase("index read");
1114
1124
  const records = [];
1115
1125
  // Passed through rather than defaulted away: an absent scope is the
1116
1126
  // caller's omission, and `walkMarkdownTree` says so.
@@ -1175,6 +1185,7 @@ export function collectContentIndex(
1175
1185
  );
1176
1186
 
1177
1187
  // Each recorded note with its body, for the words its prose cites.
1188
+ phase("index notes");
1178
1189
  const bodies = [];
1179
1190
  for (const { frontmatter, body, bodyLine, bodyColumn, raw, absPath } of parsedNotes) {
1180
1191
  // Before the index derives anything onto the frontmatter, so a later
@@ -1251,16 +1262,19 @@ export function collectContentIndex(
1251
1262
 
1252
1263
  // The words each note's prose cites, read once every language's words
1253
1264
  // are recorded; the index keeps the citing Addresses and no text.
1265
+ phase("index citations");
1254
1266
  attachCitations(records, bodies);
1255
1267
 
1256
1268
  // A run marked with a language alone takes its language's script, which
1257
1269
  // is read once every language note is recorded.
1270
+ phase("index encoding runs");
1258
1271
  fillEncodingRuns(records, { languages, contentPackage, rootDir });
1259
1272
 
1260
1273
  // Every note record exists and every Address in it is a tuple, so a parent
1261
1274
  // is found by its canonical address. Before the sort, which reads nothing
1262
1275
  // the pass writes. Calendars first: they read the culture a note states,
1263
1276
  // which the culture pass then replaces with the one it resolves.
1277
+ phase("index calendars and cultures");
1264
1278
  resolveNoteCalendars(records, {
1265
1279
  contentPackage,
1266
1280
  defaultCalendar,
@@ -1298,6 +1312,7 @@ export function collectContentIndex(
1298
1312
  // projection as the surfaces, without making the index emitter async.
1299
1313
  // Public notes resolve providers in their audience, including lineage
1300
1314
  // through hidden languages, just as their build does.
1315
+ phase("index lexical sections");
1301
1316
  const visible = records.filter((record) => !isGmNote(record));
1302
1317
  const publicSections = lexicalSectionsFor(visible, records);
1303
1318
  const allSections = lexicalSectionsFor(records);
@@ -1319,6 +1334,7 @@ export function collectContentIndex(
1319
1334
  // about the content, and the artifact is meant to be byte-identical across
1320
1335
  // two runs over an unchanged tree. Collected above, ahead of the notes —
1321
1336
  // see `resolveRole` — rather than walked a second time here.
1337
+ phase("index order");
1322
1338
  for (const record of assetRecords) {
1323
1339
  records.push(/** @type {Record<string, any>} */ (sortKeysDeep(record)));
1324
1340
  }
@@ -1344,9 +1360,11 @@ export function collectContentIndex(
1344
1360
  ) ||
1345
1361
  String(a.id ?? "").localeCompare(String(b.id ?? ""), "en"),
1346
1362
  );
1347
- return records.map((record) =>
1363
+ const decoded = records.map((record) =>
1348
1364
  decodeIndexAddresses(record, addressContext ?? { package: contentPackage }),
1349
1365
  );
1366
+ phaseEnd?.();
1367
+ return decoded;
1350
1368
  }
1351
1369
 
1352
1370
  /**
@@ -1465,6 +1483,8 @@ function dependencyAddresses(config) {
1465
1483
  * @param {import("./content-session.mjs").ContentSyntax} [opts.syntax] - Session scanner projections.
1466
1484
  * @param {Iterable<string>} [opts.nameTypes] - Bare name kinds, defaulting to configured item types.
1467
1485
  * @param {Map<string, object>} [opts.foreignIndex] - Captured dependency address view.
1486
+ * @param {ReturnType<typeof import("./content-timings.mjs").createContentTimings>} [opts.timings] -
1487
+ * Invocation stage accounting.
1468
1488
  * @returns {object[]} One record per note, plus one per documentation entry.
1469
1489
  */
1470
1490
  export function indexRecordsFor({
@@ -1478,6 +1498,7 @@ export function indexRecordsFor({
1478
1498
  syntax,
1479
1499
  nameTypes,
1480
1500
  foreignIndex,
1501
+ timings,
1481
1502
  } = {}) {
1482
1503
  const resolved = config ?? loadPackConfig();
1483
1504
  const tree = contentBase ?? resolved.paths.content;
@@ -1512,6 +1533,7 @@ export function indexRecordsFor({
1512
1533
  sources,
1513
1534
  syntax,
1514
1535
  nameTypes: nameTypes ?? resolved.itemTypes,
1536
+ timings,
1515
1537
  });
1516
1538
  }
1517
1539
 
@@ -41,6 +41,7 @@
41
41
  */
42
42
 
43
43
  import MarkdownIt from "markdown-it";
44
+ import { blockTokens } from "./markdown-blocks.mjs";
44
45
 
45
46
  import { scanBlocks, BLOCK_CONTAINERS } from "./content-blocks.mjs";
46
47
  import { scanVerse, hasColumns, verseLine, verseGroups, STANZA_BREAK } from "./content-verse.mjs";
@@ -737,9 +738,6 @@ function codeName(point) {
737
738
  return `U+${point.toString(16).toUpperCase().padStart(4, "0")}`;
738
739
  }
739
740
 
740
- /** A parser for the block structure of a div's body. */
741
- const blockParser = new MarkdownIt({ html: true });
742
-
743
741
  /** What a block token is called in a finding. */
744
742
  const BLOCK_WHAT = Object.freeze({
745
743
  heading_open: "a heading",
@@ -1485,7 +1483,7 @@ function analyseRuns(source, ctx, syntax) {
1485
1483
  { line: first + line + 1, column: /^ */.exec(body[line])[0].length + 1 },
1486
1484
  `a div marked \`${marking}\` holds paragraphs and line breaks only, not ${what}`,
1487
1485
  );
1488
- for (const token of blockParser.parse(body.join("\n"), {})) {
1486
+ for (const token of blockTokens(body.join("\n"))) {
1489
1487
  if (token.level !== 0 || !token.map) continue;
1490
1488
  const [top, past] = token.map;
1491
1489
  if (token.type === "paragraph_open") {
@@ -1535,7 +1533,14 @@ function analyseRuns(source, ctx, syntax) {
1535
1533
  const boundsOf = (detail) =>
1536
1534
  detail.reference ??
1537
1535
  (detail.run.kind === "span" ? [detail.run.start + 1, detail.contentEnd] : detail.body);
1536
+ // Each run's parent and owner are read many times while ownership is
1537
+ // assigned below, so each is found once.
1538
+ const parents = new Map();
1538
1539
  const parentOf = (detail) => {
1540
+ if (!parents.has(detail)) parents.set(detail, findParent(detail));
1541
+ return parents.get(detail);
1542
+ };
1543
+ const findParent = (detail) => {
1539
1544
  if (detail.run.kind !== "span" || Object.keys(detail.keys).length) return undefined;
1540
1545
  return details
1541
1546
  .filter(
@@ -1547,9 +1552,13 @@ function analyseRuns(source, ctx, syntax) {
1547
1552
  )
1548
1553
  .sort((a, b) => boundsOf(a)[1] - boundsOf(a)[0] - (boundsOf(b)[1] - boundsOf(b)[0]))[0];
1549
1554
  };
1555
+ const owners = new Map();
1550
1556
  const ownerOf = (detail) => {
1551
- const parent = parentOf(detail);
1552
- return parent ? ownerOf(parent) : detail;
1557
+ if (!owners.has(detail)) {
1558
+ const parent = parentOf(detail);
1559
+ owners.set(detail, parent ? ownerOf(parent) : detail);
1560
+ }
1561
+ return owners.get(detail);
1553
1562
  };
1554
1563
  const wholeVerseReading = (detail) => {
1555
1564
  if (detail.run.kind === "verse") return true;
@@ -42,7 +42,7 @@
42
42
  * @module
43
43
  */
44
44
 
45
- import MarkdownIt from "markdown-it";
45
+ import { blockTokens } from "./markdown-blocks.mjs";
46
46
 
47
47
  import { parseExtensionAttributes, refusedAttributes } from "./extension-attributes.mjs";
48
48
 
@@ -78,8 +78,6 @@ export const BOOK_SCROLL_ROWS = 40;
78
78
  /** The class every scroll carries. */
79
79
  export const SCROLL_CLASS = "scroll";
80
80
 
81
- const structureParser = new MarkdownIt({ html: true });
82
-
83
81
  /**
84
82
  * One `scroll` fence.
85
83
  *
@@ -129,7 +127,7 @@ export function scanScrolls(source) {
129
127
  if (!/^ *(?:`{3,}|~{3,}) *scroll(?:\s|$)/m.test(text)) return { blocks, errors };
130
128
  const lines = text.split("\n");
131
129
  const literalLines = new Set();
132
- for (const token of structureParser.parse(text, {})) {
130
+ for (const token of blockTokens(text)) {
133
131
  if (!token.map || !["blockquote_open", "html_block"].includes(token.type)) continue;
134
132
  for (let line = token.map[0]; line < token.map[1]; line++) literalLines.add(line);
135
133
  }
@@ -144,6 +144,7 @@ export async function createContentSession({
144
144
  problems,
145
145
  foreignIndex: foreign.index,
146
146
  syntax,
147
+ timings,
147
148
  }),
148
149
  );
149
150
  const memo = (key, make) => {
@@ -24,12 +24,15 @@ export function createContentTimings() {
24
24
  stages.set(name, (stages.get(name) ?? 0) + now - changed);
25
25
  changed = now;
26
26
  };
27
+ // Ending a stage also ends any stage opened inside it and left open.
27
28
  const begin = (name) => {
28
29
  account();
30
+ const depth = stack.length;
29
31
  stack.push(name);
30
32
  return () => {
33
+ if (stack.length <= depth) return;
31
34
  account();
32
- stack.pop();
35
+ stack.length = depth;
33
36
  };
34
37
  };
35
38
  return {
@@ -51,6 +54,15 @@ export function createContentTimings() {
51
54
  stop();
52
55
  }
53
56
  },
57
+ /**
58
+ * Open a stage that runs until the returned function is called, or
59
+ * until the stage it was opened in ends.
60
+ * @param {string} name
61
+ * @returns {() => void}
62
+ */
63
+ start(name) {
64
+ return begin(name);
65
+ },
54
66
  /** @param {string} name */
55
67
  hit(name) {
56
68
  hits.set(name, (hits.get(name) ?? 0) + 1);
@@ -180,11 +180,11 @@ function columnsMarkdown(block, lines, step) {
180
180
  }
181
181
 
182
182
  import MarkdownIt from "markdown-it";
183
+ import { blockTokens } from "./markdown-blocks.mjs";
183
184
  import { slugify } from "./content-slug.mjs";
184
185
  import { spanMarkdownPlugin } from "./content-spans.mjs";
185
186
  import { parseExtensionAttributes, refusedAttributes } from "./extension-attributes.mjs";
186
187
 
187
- const structureParser = new MarkdownIt({ html: true });
188
188
  const lineParser = new MarkdownIt({ html: true }).use(spanMarkdownPlugin);
189
189
 
190
190
  /** The structural first span of a line, with its source ranges and keys. */
@@ -223,7 +223,7 @@ export function scanVerse(source) {
223
223
  blocks = [],
224
224
  errors = [];
225
225
  const literalLines = new Set();
226
- for (const token of structureParser.parse(String(source ?? ""), {})) {
226
+ for (const token of blockTokens(source)) {
227
227
  if (!token.map || !["blockquote_open", "html_block"].includes(token.type)) continue;
228
228
  for (let line = token.map[0]; line < token.map[1]; line++) literalLines.add(line);
229
229
  }
@@ -464,6 +464,16 @@ export function dictionaryAndTermRows(records, all = records) {
464
464
  );
465
465
  return hit ? hit.entry[key] : undefined;
466
466
  };
467
+ // One piece lookup per language, shared by every word that language states.
468
+ const glossers = new Map();
469
+ const glossesFor = (language) => {
470
+ if (!glossers.has(language))
471
+ glossers.set(
472
+ language,
473
+ pieceGlosses(...lineage(language).map((key) => languages.get(key).data)),
474
+ );
475
+ return glossers.get(language);
476
+ };
467
477
  const etymologyOf = ({ word, statedIn, depth }) => {
468
478
  const blank = Object.fromEntries(ETYMOLOGY_KEYS.map((key) => [key, null]));
469
479
  if (depth > 0)
@@ -476,13 +486,15 @@ export function dictionaryAndTermRows(records, all = records) {
476
486
  };
477
487
  const source = isMap(word.source) ? word.source : {};
478
488
  const built = Array.isArray(word.built) ? word.built.map(String) : [];
479
- const glossOf = pieceGlosses(...lineage(statedIn).map((key) => languages.get(key).data));
480
489
  const out = {
481
490
  ...blank,
482
491
  kind: typeof source.kind === "string" ? source.kind : null,
483
492
  language: typeof source.language === "string" ? source.language : null,
484
493
  form: typeof source.form === "string" ? source.form : null,
485
- parts: built.length ? built.map((form) => ({ form, gloss: glossOf(form) })) : null,
494
+ parts:
495
+ built.length ?
496
+ built.map((form) => ({ form, gloss: glossesFor(statedIn)(form) }))
497
+ : null,
486
498
  root: typeof word.root === "string" ? word.root : null,
487
499
  frame: typeof word.frame === "string" ? word.frame : null,
488
500
  };
@@ -69,7 +69,14 @@ function capitalWriting(enc, word, native, font) {
69
69
  upper.to + native.slice(lower.to.length)
70
70
  : undefined;
71
71
  }
72
+ /** Each lexicon's forms, found once; a lexicon is not modified once it is converted with. */
73
+ const formsOf = new WeakMap();
72
74
  function entryForms(lexicon) {
75
+ if (!lexicon) return findEntryForms(lexicon);
76
+ if (!formsOf.has(lexicon)) formsOf.set(lexicon, findEntryForms(lexicon));
77
+ return formsOf.get(lexicon);
78
+ }
79
+ function findEntryForms(lexicon) {
73
80
  const data = lexicon?.data ?? {};
74
81
  const derived = new Map();
75
82
  for (const entry of derivedForms(data)) {
@@ -122,6 +129,44 @@ function entryForms(lexicon) {
122
129
  ];
123
130
  }
124
131
 
132
+ /** Each lexicon's phrases of more than one word, with the patterns that find them. */
133
+ const phrasesFound = new WeakMap();
134
+ function phrasesOf(lexicon) {
135
+ if (!phrasesFound.has(lexicon))
136
+ phrasesFound.set(
137
+ lexicon,
138
+ entryForms(lexicon)
139
+ .map((entry) => entry.form)
140
+ .filter((form) => [...form.matchAll(WORD)].length > 1)
141
+ .map((form) => ({
142
+ form,
143
+ spaces: form.match(/[\p{Zs}\u0009-\u000D]+/gu) ?? [],
144
+ pattern: new RegExp(
145
+ form
146
+ .split(/[\p{Zs}\u0009-\u000D]+/u)
147
+ .map((part) => part.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"))
148
+ .join("[\\p{Zs}\\u0009-\\u000D]+"),
149
+ "iuy",
150
+ ),
151
+ })),
152
+ );
153
+ return phrasesFound.get(lexicon);
154
+ }
155
+
156
+ /** Each lexicon's words' forms and variants, as written. */
157
+ const statedFound = new WeakMap();
158
+ function statedForms(lexicon) {
159
+ if (!statedFound.has(lexicon)) {
160
+ const forms = new Set();
161
+ for (const entry of list(lexicon.data.words)) {
162
+ forms.add(entry.form);
163
+ for (const variant of list(entry.variants)) forms.add(variant);
164
+ }
165
+ statedFound.set(lexicon, forms);
166
+ }
167
+ return statedFound.get(lexicon);
168
+ }
169
+
125
170
  function literalEnd(text, at) {
126
171
  let cursor = at + OPEN.length;
127
172
  while (cursor < text.length) {
@@ -164,23 +209,7 @@ export function encodeWithLanguage(enc, text, options = {}) {
164
209
  }
165
210
  const writing =
166
211
  options.lexicon ? writingsOf(options.lexicon.data, enc.name, options.font) : undefined;
167
- const phrases =
168
- options.lexicon ?
169
- entryForms(options.lexicon)
170
- .map((entry) => entry.form)
171
- .filter((form) => [...form.matchAll(WORD)].length > 1)
172
- .map((form) => ({
173
- form,
174
- spaces: form.match(/[\p{Zs}\u0009-\u000D]+/gu) ?? [],
175
- pattern: new RegExp(
176
- form
177
- .split(/[\p{Zs}\u0009-\u000D]+/u)
178
- .map((part) => part.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"))
179
- .join("[\\p{Zs}\\u0009-\\u000D]+"),
180
- "iuy",
181
- ),
182
- }))
183
- : [];
212
+ const phrases = options.lexicon ? phrasesOf(options.lexicon) : [];
184
213
  let at = 0;
185
214
  const appendRows = (from, to) => {
186
215
  let cursor = from;
@@ -267,9 +296,7 @@ export function encodeWithLanguage(enc, text, options = {}) {
267
296
  } else if (own && "text" in own) {
268
297
  appendRows(at, start);
269
298
  let native = own.text;
270
- const exact = list(options.lexicon.data.words).some(
271
- (entry) => entry.form === lookup || list(entry.variants).includes(lookup),
272
- );
299
+ const exact = statedForms(options.lexicon).has(lookup);
273
300
  if (!exact && match[0] !== match[0].toLowerCase()) {
274
301
  const capital =
275
302
  match[0] === initialCapital(match[0].toLowerCase()) ?
@@ -22,6 +22,7 @@ import { isLanguageRecord, lineageChain, noteAddressOf } from "./word-rows.mjs";
22
22
  import { derivedForms, foldWord } from "./word-forms.mjs";
23
23
  import { WRITTEN_ENDINGS, RESERVED_TONGUE } from "./language-terms.mjs";
24
24
  import { LANGUAGE_SUBTYPE, glossText } from "./word-terms.mjs";
25
+ import { ENGLISH_GROUP } from "./language-names.mjs";
25
26
  import { writingsOf } from "./language-writings.mjs";
26
27
  import { encodeText, validateSource, EncodingError } from "./encoding.mjs";
27
28
  import { encodeWithLanguage } from "./encoding-language.mjs";
@@ -389,9 +390,21 @@ export function nativeRows(records) {
389
390
  return rows;
390
391
  }
391
392
 
392
- /** Project only conversion-readable language data to a JSON-safe object. */
393
+ /** Each lexicon's projection, made once and shared by every conversion that reads it. */
394
+ const serialized = new WeakMap();
395
+
396
+ /**
397
+ * Project only conversion-readable language data to a JSON-safe object. The
398
+ * projection of one lexicon is made once and shared, so it must not be modified.
399
+ */
393
400
  export function serializeLexicon(lexicon) {
394
401
  if (!lexicon) return undefined;
402
+ if (!serialized.has(lexicon)) serialized.set(lexicon, projectLexicon(lexicon));
403
+ return serialized.get(lexicon);
404
+ }
405
+
406
+ /** The conversion-readable data of a lexicon, copied. */
407
+ function projectLexicon(lexicon) {
395
408
  const select = (value, keys) =>
396
409
  Object.fromEntries(
397
410
  keys
@@ -630,7 +643,10 @@ export function checkEncodingLexicon(note, ctx) {
630
643
  ...new Map(
631
644
  [...(ctx.encodings?.values() ?? [])].map((encoding) => [encoding.name, encoding]),
632
645
  ).values(),
633
- ].filter((encoding) => aliases.has(encoding.language));
646
+ ]
647
+ .filter((encoding) => aliases.has(encoding.language))
648
+ // A codes-only script accepts no letter, so the roman forms are not written in it.
649
+ .filter((encoding) => [...encoding.accepted].some((char) => /\p{L}/u.test(char)));
634
650
  if (!encodings.length) return [];
635
651
  const forms = [],
636
652
  data = note.fm.data ?? {};
@@ -641,6 +657,7 @@ export function checkEncodingLexicon(note, ctx) {
641
657
  add(word?.form, ["words", i, "form"]);
642
658
  listOf(word?.variants).forEach((form, j) => add(form, ["words", i, "variants", j]));
643
659
  });
660
+ // `en` holds English names, which no encoding of the tongue sees.
644
661
  const names = (value, keys) => {
645
662
  if (Array.isArray(value))
646
663
  value.forEach((item, i) =>
@@ -648,7 +665,12 @@ export function checkEncodingLexicon(note, ctx) {
648
665
  );
649
666
  else if (isMap(value))
650
667
  for (const [key, item] of Object.entries(value))
651
- if (key !== "pattern" && key !== "metadata") names(item, [...keys, key]);
668
+ if (
669
+ key !== "pattern" &&
670
+ key !== "metadata" &&
671
+ !(keys.length === 1 && key === ENGLISH_GROUP)
672
+ )
673
+ names(item, [...keys, key]);
652
674
  };
653
675
  names(data.names, ["names"]);
654
676
  listOf(data.elements).forEach((element, i) => {
@@ -34,15 +34,12 @@
34
34
  */
35
35
 
36
36
  import { parseExtensionAttributes, refusedAttributes } from "./extension-attributes.mjs";
37
- import MarkdownIt from "markdown-it";
38
-
39
- /** Literal block ranges use the renderer's Markdown grammar. */
40
- const structureParser = new MarkdownIt({ html: true });
37
+ import { blockTokens } from "./markdown-blocks.mjs";
41
38
 
42
39
  /** Body lines occupied by code or raw HTML, including comment examples. */
43
40
  function literalHeadingLines(source) {
44
41
  const lines = new Set();
45
- for (const token of structureParser.parse(String(source ?? ""), {})) {
42
+ for (const token of blockTokens(source)) {
46
43
  if (!token.map || !["fence", "code_block", "html_block"].includes(token.type)) continue;
47
44
  for (let line = token.map[0]; line < token.map[1]; line++) lines.add(line);
48
45
  }
@@ -33,6 +33,10 @@
33
33
  * person's gender's list, or from every given list together where that gender
34
34
  * has none; `{given.<gender>}` draws from that gender's list whatever the
35
35
  * person's gender, as a patronymic draws a father's name.
36
+ * - `en` — English names, in groups of lists: `en.epithet` is a list a pattern
37
+ * draws from as `{en.epithet}`, as `"{given} the {en.epithet}"` does. English
38
+ * names are never encoded in the tongue's script or checked against its
39
+ * spelling.
36
40
  * - every other key — a list of names the pattern draws from by that key:
37
41
  * `clan`, `family`, `place`. A list name is letters, digits and underscores,
38
42
  * beginning with a letter.
@@ -55,13 +59,16 @@ import { nearest } from "./near-miss.mjs";
55
59
  /** The `skill` subType whose notes may carry name lists. */
56
60
  export const NAMES_SUBTYPE = "language";
57
61
 
62
+ /** The key whose groups of lists hold English names. */
63
+ export const ENGLISH_GROUP = "en";
64
+
58
65
  /** The placeholder that draws from the given-name list for a gender. */
59
66
  export const GIVEN_LIST = "given";
60
67
 
61
68
  /** What a list name may be spelled with. */
62
69
  const LIST_NAME = /^[A-Za-z][A-Za-z0-9_]*$/;
63
70
 
64
- /** A placeholder's text: a list name, and for `given` an optional gender. */
71
+ /** A placeholder's text: a list name, and after a dot a gender for `given` or a list for `en`. */
65
72
  const PLACEHOLDER = /^([A-Za-z][A-Za-z0-9_]*)(?:\.([A-Za-z][A-Za-z0-9_]*))?$/;
66
73
 
67
74
  /**
@@ -80,11 +87,12 @@ export function requiredGivenGenders() {
80
87
  * One piece of a parsed pattern: literal text, or a placeholder naming a list
81
88
  * and, for `given`, the gender whose list it draws from.
82
89
  *
83
- * @typedef {{literal: string} | {list: string, gender?: string}} PatternPart
90
+ * @typedef {{literal: string} | {list: string, gender?: string, sub?: string}} PatternPart
84
91
  */
85
92
 
86
93
  /** A placeholder as written, without its braces. */
87
94
  function placeholderText(part) {
95
+ if (part.sub !== undefined) return `${part.list}.${part.sub}`;
88
96
  return part.gender ? `${part.list}.${part.gender}` : part.list;
89
97
  }
90
98
 
@@ -126,17 +134,22 @@ export function parsePattern(pattern) {
126
134
  `has the placeholder \`{${text}}\`; a placeholder names a list in letters, ` +
127
135
  "digits and underscores, beginning with a letter",
128
136
  );
129
- else if (match[2] !== undefined && match[1] !== GIVEN_LIST)
137
+ else if (match[1] === ENGLISH_GROUP && match[2] === undefined)
138
+ problems.push(
139
+ `has the placeholder \`{${text}}\`; \`en\` holds groups of English names, ` +
140
+ "so a placeholder names one as `{en.<list>}`",
141
+ );
142
+ else if (match[2] !== undefined && match[1] !== GIVEN_LIST && match[1] !== ENGLISH_GROUP)
130
143
  problems.push(
131
144
  `has the placeholder \`{${text}}\`; only \`given\` takes a gender after a dot, ` +
132
- "as `{given.male}` does",
145
+ "as `{given.male}` does, and `en` a list, as `{en.epithet}` does",
133
146
  );
134
147
  else {
135
148
  if (literal) parts.push({ literal });
136
149
  literal = "";
137
- parts.push(
138
- match[2] === undefined ? { list: match[1] } : { list: match[1], gender: match[2] },
139
- );
150
+ if (match[2] === undefined) parts.push({ list: match[1] });
151
+ else if (match[1] === ENGLISH_GROUP) parts.push({ list: match[1], sub: match[2] });
152
+ else parts.push({ list: match[1], gender: match[2] });
140
153
  }
141
154
  i = close + 1;
142
155
  }
@@ -263,7 +276,30 @@ export function namesProblems(names, lookVocabulary) {
263
276
  });
264
277
  };
265
278
 
266
- const lists = Object.keys(names).filter((key) => key !== "pattern");
279
+ const lists = Object.keys(names).filter((key) => key !== "pattern" && key !== ENGLISH_GROUP);
280
+ const english = names[ENGLISH_GROUP];
281
+ const englishLists = [];
282
+ if (english !== undefined && english !== null) {
283
+ if (!isPlainMap(english))
284
+ shape(
285
+ [ENGLISH_GROUP],
286
+ `\`${label([ENGLISH_GROUP])}\` should be a map of names to lists of English names`,
287
+ );
288
+ else
289
+ for (const key of Object.keys(english)) {
290
+ if (!LIST_NAME.test(key)) {
291
+ problem(
292
+ [ENGLISH_GROUP, key],
293
+ `"${key}" is not a list name; a list of \`${label([ENGLISH_GROUP])}\` is ` +
294
+ "named in letters, digits and underscores, beginning with a letter",
295
+ true,
296
+ );
297
+ continue;
298
+ }
299
+ englishLists.push(key);
300
+ checkList([ENGLISH_GROUP, key]);
301
+ }
302
+ }
267
303
  for (const key of lists) {
268
304
  if (key === GIVEN_LIST) continue;
269
305
  if (!LIST_NAME.test(key)) {
@@ -376,6 +412,19 @@ export function namesProblems(names, lookVocabulary) {
376
412
  if (parsed.problems.length) unreadable = true;
377
413
  for (const part of parsed.parts) {
378
414
  if (!("list" in part)) continue;
415
+ if (part.sub !== undefined) {
416
+ named.add(placeholderText(part));
417
+ if (englishLists.includes(part.sub)) continue;
418
+ const guess = nearest(part.sub, englishLists);
419
+ problem(
420
+ path,
421
+ `\`{${placeholderText(part)}}\` in \`${label(path)}\` names no list of ` +
422
+ `\`${label([ENGLISH_GROUP])}\`` +
423
+ (englishLists.length ? `; the lists are ${englishLists.join(", ")}` : "") +
424
+ (guess ? `. Did you mean "{en.${guess}}"?` : ""),
425
+ );
426
+ continue;
427
+ }
379
428
  named.add(part.list);
380
429
  if (part.gender !== undefined) {
381
430
  if (givenMap[part.gender] === undefined || givenMap[part.gender] === null)
@@ -397,6 +446,15 @@ export function namesProblems(names, lookVocabulary) {
397
446
  }
398
447
  }
399
448
  if (unreadable) return problems;
449
+ for (const list of englishLists) {
450
+ if (named.has(`${ENGLISH_GROUP}.${list}`)) continue;
451
+ problem(
452
+ [ENGLISH_GROUP, list],
453
+ `\`${label([ENGLISH_GROUP, list])}\` is never drawn from: no pattern of ` +
454
+ `\`${label(["pattern"])}\` has \`{${ENGLISH_GROUP}.${list}}\``,
455
+ true,
456
+ );
457
+ }
400
458
  for (const list of lists) {
401
459
  if (named.has(list)) continue;
402
460
  problem(
@@ -490,6 +548,7 @@ export function givenNames(names, gender) {
490
548
 
491
549
  /** The names one placeholder draws from, for a person of `gender`. */
492
550
  function listFor(names, part, gender) {
551
+ if (part.sub !== undefined) return names[part.list][part.sub];
493
552
  if (part.list !== GIVEN_LIST) return names[part.list];
494
553
  if (part.gender !== undefined) return names[GIVEN_LIST][part.gender];
495
554
  return givenNames(names, gender);