@furifont/core 0.3.0 → 0.4.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (3) hide show
  1. package/dist/index.d.mts +159 -140
  2. package/dist/index.mjs +546 -369
  3. package/package.json +1 -1
package/dist/index.mjs CHANGED
@@ -10,20 +10,8 @@ const DEFAULTS = {
10
10
  plan: {
11
11
  family: "Furifont Sans",
12
12
  rubyScript: "hiragana",
13
- /** Newspaper frequency buckets, nf01 to nf48: the window of words that get ruby. All of them: it adds about 400 glyphs. */
14
- nfMin: 1,
15
- nfMax: 48,
16
- /** JMdict priority tags that select a word whatever its bucket: the first tier of each list, which JMdict marks as common. */
17
- selectTags: [
18
- "ichi1",
19
- "news1",
20
- "spec1",
21
- "gai1"
22
- ],
23
- /** Uses in Tatoeba's example sentences that select a word whatever its bucket: about 1,000 more words, and 50 glyphs. */
24
- minTatoeba: 3,
25
- places: true,
26
- untagged: true,
13
+ /** No word is left bare for being common; every word of every source gets ruby. */
14
+ skip: 0,
27
15
  isolated: true,
28
16
  isolatedMinShare: .8,
29
17
  /** Sizes in em, relative to the base font. */
@@ -82,8 +70,8 @@ const PREBUILT = {
82
70
  style: "VF",
83
71
  weights: [250, 900]
84
72
  },
85
- /** Options set on every prebuilt font over the defaults above, such as `{ nfMax: 24 }`. */
86
- options: {}
73
+ /** Options set on every prebuilt font over the defaults above, such as `{ take: 20000 }`. */
74
+ options: { take: 4e4 }
87
75
  };
88
76
  /**
89
77
  * The sequence, 。 and VS1, a page measures to tell which version of Furifont
@@ -101,16 +89,22 @@ const HAN = /\p{Script=Han}/u;
101
89
  function isHan(char) {
102
90
  return HAN.test(char);
103
91
  }
92
+ /** Spans of a JmdictFurigana alignment; a loop, not splits, as the planner parses the 400,000 readings of the dictionary as it starts. */
104
93
  function parseFurigana(furigana) {
105
- return furigana.split(";").filter(Boolean).map((part) => {
106
- const [range, ruby = ""] = part.split(":");
107
- const [start, end = start] = range.split("-").map(Number);
108
- return {
94
+ const spans = [];
95
+ for (const part of furigana.split(";")) {
96
+ if (!part) continue;
97
+ const colon = part.indexOf(":");
98
+ const range = colon < 0 ? part : part.slice(0, colon);
99
+ const dash = range.indexOf("-");
100
+ const start = Number(dash < 0 ? range : range.slice(0, dash));
101
+ spans.push({
109
102
  start,
110
- end: end + 1,
111
- ruby
112
- };
113
- });
103
+ end: (dash < 0 ? start : Number(range.slice(dash + 1))) + 1,
104
+ ruby: colon < 0 ? "" : part.slice(colon + 1)
105
+ });
106
+ }
107
+ return spans;
114
108
  }
115
109
  /**
116
110
  * Splits a form into its Han runs. Browsers itemize text by script before
@@ -179,6 +173,413 @@ function formatNotation(text, spans) {
179
173
  }
180
174
  return out + chars.slice(at).join("");
181
175
  }
176
+ function getOrCreate(map, key, create) {
177
+ let value = map.get(key);
178
+ if (value === void 0) map.set(key, value = create());
179
+ return value;
180
+ }
181
+ //#endregion
182
+ //#region src/schema.ts
183
+ const RUBY_SCRIPTS = [
184
+ "hiragana",
185
+ "katakanaOn",
186
+ "hepburn"
187
+ ];
188
+ /** The dictionary's manually maintained sources. */
189
+ const MANUAL_SOURCES = [
190
+ "mahjong-yaku",
191
+ "jr-stations",
192
+ "yojijukugo",
193
+ "japan-post"
194
+ ];
195
+ /**
196
+ * The sources a plan takes words from: any JMdict word; JMdict's priority
197
+ * lists (both tiers of each) and its newspaper ranking; words Tatoeba's
198
+ * sentences use at least three times; JMnedict's place names; the manual lists.
199
+ */
200
+ const WORD_SOURCES = [
201
+ "jmdict",
202
+ "ichi",
203
+ "news",
204
+ "spec",
205
+ "gai",
206
+ "nf",
207
+ "tatoeba",
208
+ "place",
209
+ ...MANUAL_SOURCES
210
+ ];
211
+ /** Easiest first. */
212
+ const JLPT_LEVELS = [
213
+ "N5",
214
+ "N4",
215
+ "N3",
216
+ "N2",
217
+ "N1"
218
+ ];
219
+ /**
220
+ * Every source a reading may have, a bit each in this order: the word
221
+ * sources, then the easiest JLPT level listing it, `inflecting` (a kanji read
222
+ * before okurigana in a JMdict word no list tags, such as 奉る) and `default`,
223
+ * the dictionary's curated default.
224
+ */
225
+ const SOURCES = [
226
+ ...WORD_SOURCES,
227
+ ...JLPT_LEVELS,
228
+ "inflecting",
229
+ "default"
230
+ ];
231
+ const sourceBit = (source) => 1 << SOURCES.indexOf(source);
232
+ /** The sources a mask of `SOURCES` bits names. */
233
+ const sourcesIn = (mask) => SOURCES.filter((source) => mask & sourceBit(source));
234
+ /** Sizes in em, relative to the base font. */
235
+ const RubyMetricsSchema = v.object({
236
+ /** Ruby size relative to the base characters. */
237
+ rubyScale: v.optional(v.pipe(v.number(), v.minValue(.2), v.maxValue(.8)), DEFAULTS.plan.metrics.rubyScale),
238
+ /** Space between the top of the em box and the bottom of the ruby. */
239
+ rubyGap: v.optional(v.pipe(v.number(), v.minValue(0), v.maxValue(.5)), DEFAULTS.plan.metrics.rubyGap),
240
+ /** Ruby is squeezed horizontally down to this scale before it may overhang. */
241
+ compressionFloor: v.optional(v.pipe(v.number(), v.minValue(.3), v.maxValue(1)), DEFAULTS.plan.metrics.compressionFloor),
242
+ /** Widest a ruby may get, overhanging its base symmetrically. */
243
+ maxRubyWidth: v.optional(v.pipe(v.number(), v.minValue(1), v.maxValue(3)), DEFAULTS.plan.metrics.maxRubyWidth)
244
+ });
245
+ const Count = v.pipe(v.number(), v.integer(), v.minValue(0));
246
+ const PlanOptionsSchema = v.object({
247
+ family: v.optional(v.pipe(v.string(), v.trim(), v.nonEmpty(), v.check((name) => !/source/i.test(name), "Source is a reserved font name of the base font")), DEFAULTS.plan.family),
248
+ rubyScript: v.optional(v.picklist(RUBY_SCRIPTS), DEFAULTS.plan.rubyScript),
249
+ /** The sources whose words get ruby. */
250
+ sources: v.optional(v.array(v.picklist(WORD_SOURCES)), () => [...WORD_SOURCES]),
251
+ /** The most common words left bare, by score: a reader knows them. Lone kanji count too. */
252
+ skip: v.optional(Count, DEFAULTS.plan.skip),
253
+ /** How many words get ruby after those skipped, by score; all of them when left out. */
254
+ take: v.optional(v.pipe(Count, v.minValue(1))),
255
+ /** Words on the JLPT lists of this level and every easier one stay bare. */
256
+ skipJlpt: v.optional(v.picklist(JLPT_LEVELS)),
257
+ /** Annotate a kanji alone in its run, or one no word claims after another kanji (a suffix); when off, only selectors give them ruby. */
258
+ isolated: v.optional(v.boolean(), DEFAULTS.plan.isolated),
259
+ /** A kanji alone in its run is annotated only when one reading has this share of the evidence. */
260
+ isolatedMinShare: v.optional(v.pipe(v.number(), v.minValue(.5), v.maxValue(1)), DEFAULTS.plan.isolatedMinShare),
261
+ words: v.optional(v.object({
262
+ /** Extra words in inline notation, `日[に]本[ほん]`. */
263
+ add: v.optional(v.array(v.string()), []),
264
+ /** Written forms never to annotate. */
265
+ remove: v.optional(v.array(v.string()), []),
266
+ /** Readings to make the default, in inline notation. */
267
+ pin: v.optional(v.array(v.string()), [])
268
+ }), {}),
269
+ metrics: v.optional(RubyMetricsSchema, {})
270
+ });
271
+ function resolveOptions(input = {}) {
272
+ return v.parse(PlanOptionsSchema, input);
273
+ }
274
+ const Codepoint = v.pipe(v.number(), v.integer(), v.minValue(0), v.maxValue(1114111));
275
+ /**
276
+ * The contract between the planner and the compiler. The compiler makes no
277
+ * decision about readings: it draws what `glyphs` describes and compiles `fea`.
278
+ * In `fea`, base characters are named `uniXXXX` / `uXXXXX` after their codepoint.
279
+ */
280
+ const PlanSchema = v.object({
281
+ version: v.literal(1),
282
+ names: v.object({
283
+ family: v.string(),
284
+ description: v.string()
285
+ }),
286
+ metrics: v.object({
287
+ rubyScale: v.number(),
288
+ rubyGap: v.number(),
289
+ compressionFloor: v.number(),
290
+ maxRubyWidth: v.number()
291
+ }),
292
+ /** Base characters the rules name; each must be in the base font's cmap. */
293
+ codepoints: v.array(Codepoint),
294
+ /**
295
+ * Base + variation selector sequences to map in `cmap` format 14, each to an
296
+ * empty glyph named `name`, with the base's advance: the rules replace it.
297
+ */
298
+ variants: v.array(v.object({
299
+ name: v.pipe(v.string(), v.regex(/^[A-Z_][\w.]{0,62}$/i)),
300
+ base: Codepoint,
301
+ selector: Codepoint
302
+ })),
303
+ glyphs: v.array(v.object({
304
+ name: v.pipe(v.string(), v.regex(/^[A-Z_][\w.]{0,62}$/i)),
305
+ /** The base characters drawn side by side; more than one makes a ligature. */
306
+ bases: v.pipe(v.array(Codepoint), v.minLength(1)),
307
+ /** Ruby text drawn above characters `[start, end)` of `bases`. */
308
+ ruby: v.array(v.object({
309
+ start: v.number(),
310
+ end: v.number(),
311
+ text: v.pipe(v.string(), v.nonEmpty())
312
+ }))
313
+ })),
314
+ fea: v.string(),
315
+ /** Subset the base font to these codepoints before building (live previews). */
316
+ subset: v.optional(v.array(Codepoint)),
317
+ /** Shaping checks the compiler runs on its output: `text` must shape to `glyphs`. */
318
+ smoke: v.array(v.object({
319
+ text: v.string(),
320
+ glyphs: v.array(v.string())
321
+ }))
322
+ });
323
+ /**
324
+ * The bounds of a reading's score, 0 to 100: above `tagged`, a word JMdict's
325
+ * lists tag, by its weight; above `used`, one Tatoeba's sentences use (20 and
326
+ * its uses, up to 50); above `listed`, one a manual list (15) or JMnedict (10)
327
+ * names; any other JMdict word scores `listed`.
328
+ */
329
+ const SCORE = {
330
+ tagged: 50,
331
+ used: 20,
332
+ listed: 5
333
+ };
334
+ /** Score points per factor e of weight: a point is a twelfth, about 9%, of it. */
335
+ const POINTS = 12;
336
+ /**
337
+ * The score of a tagged reading of corpus weight `weight`, at most 1: 1 / its
338
+ * newspaper bucket, or a lone kanji's share of its uses. It is 100 for 1, 92
339
+ * for 1/2, 54 for 1/48; a weight too small to score above `tagged` scores 0.
340
+ */
341
+ function scoreOf(weight) {
342
+ const score = Math.round(100 + POINTS * Math.log(weight));
343
+ return score > SCORE.tagged ? score : 0;
344
+ }
345
+ /** The corpus weight of a reading's score, which `scoreOf` gives; 0 for one no list tags. */
346
+ const weightOf = (score) => score > SCORE.tagged ? Math.exp((score - 100) / POINTS) : 0;
347
+ const isRecord = (value) => typeof value === "object" && value !== null && !Array.isArray(value);
348
+ const isReading = (value) => Array.isArray(value) && value.length === 2 && value.every(Number.isInteger) && value[0] >= 0 && value[0] <= 100 && value[1] > 0 && value[1] < 1 << SOURCES.length;
349
+ /** Readings by run or kanji, then by alignment or reading. Checked by hand: valibot takes seconds over the 400,000 readings of the dictionary. */
350
+ const ReadingsSchema = v.custom((input) => isRecord(input) && Object.values(input).every((readings) => isRecord(readings) && Object.values(readings).every(isReading)), "Expected readings by run, each [score, sources]");
351
+ /** The bit of each source, which the file writes for its readers. */
352
+ const SOURCE_BITS = Object.fromEntries(SOURCES.map((source) => [source, sourceBit(source)]));
353
+ /**
354
+ * The compiled dictionary, the contract between `@furifont/dictionary` and
355
+ * the planner: readings by the Han run a font sees, each with its score and
356
+ * sources. A run's readings come ranked, the default first. Run readings are
357
+ * JmdictFurigana alignments within the run.
358
+ */
359
+ const DictionarySchema = v.object({
360
+ /** Source datasets and the version each was compiled from. */
361
+ versions: v.record(v.string(), v.string()),
362
+ /** The bit of each source in the readings' masks. */
363
+ sources: v.custom((input) => isRecord(input) && JSON.stringify(input) === JSON.stringify(SOURCE_BITS), "Expected the sources of this version of @furifont/core"),
364
+ /** Kanji to its on'yomi, in hiragana. */
365
+ onyomi: v.record(v.string(), v.array(v.string())),
366
+ /** Runs of two or more kanji, then their alignments: `{ 日本: { '0-1:にほん': [75, 61] } }`. */
367
+ runs: ReadingsSchema,
368
+ /** Kanji alone in a run, then their readings: `{ 来: { く: [98, 2101] } }`. */
369
+ kanji: ReadingsSchema,
370
+ /** Written forms never to annotate, such as 月, whose reading alone is a guess; users' removals add to them. */
371
+ bare: v.array(v.string())
372
+ });
373
+ /** The file `DictionarySchema` reads: an entry a line. */
374
+ function formatDictionary(dictionary) {
375
+ const lines = (table) => Object.entries(table).map(([key, value]) => `${JSON.stringify(key)}:${JSON.stringify(value)}`).join(",\n");
376
+ return `{
377
+ "versions":${JSON.stringify(dictionary.versions)},
378
+ "sources":${JSON.stringify(dictionary.sources)},
379
+ "bare":${JSON.stringify(dictionary.bare)},
380
+ "onyomi":{
381
+ ${lines(dictionary.onyomi)}
382
+ },
383
+ "kanji":{
384
+ ${lines(dictionary.kanji)}
385
+ },
386
+ "runs":{
387
+ ${lines(dictionary.runs)}
388
+ }
389
+ }
390
+ `;
391
+ }
392
+ //#endregion
393
+ //#region src/fuse.ts
394
+ /** JMdict orders readings within a form; a run read inside a longer form ranks after the form's own readings. */
395
+ const OTHER_FORM_ORDER = 1e3;
396
+ const OKURIGANA = /[\u3041-\u3096]/;
397
+ /** Uses in Tatoeba's sentences that make a word one of the sentences'. */
398
+ const MIN_USES = 3;
399
+ /** With this many uses alone in the sentences, they decide how a lone kanji reads rather than the word lists. */
400
+ const LONE_MIN_USES = 20;
401
+ /** A tagged reading of a kanji that the sentences decide keeps this score at most, so it stays among the kanji's readings. */
402
+ const LONE_TAGGED_FLOOR = 60;
403
+ const PRIORITY_LISTS = [
404
+ "ichi",
405
+ "news",
406
+ "spec",
407
+ "gai"
408
+ ];
409
+ const MANUAL_BITS = MANUAL_SOURCES.reduce((mask, name) => mask | sourceBit(name), 0);
410
+ /** The evidence of `text` read as `key`, met for the first time or not. */
411
+ function evidenceOf(table, text, key) {
412
+ return getOrCreate(getOrCreate(table, text, () => /* @__PURE__ */ new Map()), key, () => ({ sources: 0 }));
413
+ }
414
+ /** Spans as a JmdictFurigana alignment: `0:に;1:ほん`, `0-1:きょう`. */
415
+ function formatFurigana(spans) {
416
+ return spans.map((s) => `${s.end - s.start > 1 ? `${s.start}-${s.end - 1}` : s.start}:${s.ruby}`).join(";");
417
+ }
418
+ /** Rough corpus frequency of a tagged entry, in newspaper buckets of ~500 words each. */
419
+ function rankOf(entry) {
420
+ if (entry.priority) return entry.nf || Number(/nf(\d+)/.exec(entry.priority)?.[1] ?? 0) || (/[a-z]1\b/.test(entry.priority) ? 24 : 48);
421
+ }
422
+ /** Adds what a JMdict entry says of a reading: its lists, its rank and its JLPT level. */
423
+ function addEntry(into, entry) {
424
+ into.sources |= sourceBit("jmdict");
425
+ if (entry.jlpt) into.jlpt = Math.max(into.jlpt ?? 0, entry.jlpt);
426
+ const rank = rankOf(entry);
427
+ if (!rank) return;
428
+ into.rank = Math.min(into.rank ?? rank, rank);
429
+ if (entry.nf) into.sources |= sourceBit("nf");
430
+ for (const list of PRIORITY_LISTS.filter((list) => new RegExp(`\\b${list}[12]\\b`).test(entry.priority))) into.sources |= sourceBit(list);
431
+ }
432
+ function addRun(runs, run, entry) {
433
+ const evidence = evidenceOf(runs, run.text, formatFurigana(run.spans));
434
+ const order = run.text === entry.form ? entry.order : OTHER_FORM_ORDER + entry.order;
435
+ evidence.order = Math.min(evidence.order ?? order, order);
436
+ if (entry.tatoeba) evidence.uses = Math.max(evidence.uses ?? 0, entry.tatoeba);
437
+ addEntry(evidence, entry);
438
+ }
439
+ function addKanji(kanji, run, entry) {
440
+ const ruby = run.spans[0].ruby;
441
+ if (entry.tatoeba && !entry.conjugatesReading) {
442
+ const uses = getOrCreate(kanji.uses, run.text, () => /* @__PURE__ */ new Map());
443
+ uses.set(ruby, (uses.get(ruby) ?? 0) + entry.tatoeba);
444
+ }
445
+ if (rankOf(entry)) for (const r of entry.conjugatesReading && ruby === "く" ? [
446
+ "く",
447
+ "き",
448
+ "こ"
449
+ ] : [ruby]) addEntry(evidenceOf(kanji.tagged, run.text, r), entry);
450
+ else if (OKURIGANA.test(entry.form[entry.form.indexOf(run.text) + run.text.length] ?? "")) getOrCreate(kanji.inflecting, run.text, () => /* @__PURE__ */ new Set()).add(ruby);
451
+ }
452
+ /** Its sources with the JLPT level and Tatoeba's sentences as bits. */
453
+ function sourcesOf(evidence) {
454
+ const level = evidence.jlpt && JLPT_LEVELS[5 - evidence.jlpt];
455
+ return evidence.sources | (level ? sourceBit(level) : 0) | ((evidence.uses ?? 0) >= MIN_USES ? sourceBit("tatoeba") : 0);
456
+ }
457
+ /** How common a run's reading is, by the most telling of its sources. */
458
+ function runScore(evidence) {
459
+ const uses = evidence.uses ?? 0;
460
+ return Math.max(evidence.rank ? scoreOf(1 / evidence.rank) : 0, uses >= MIN_USES ? SCORE.used + Math.min(uses, SCORE.tagged - SCORE.used) : 0, evidence.sources & MANUAL_BITS ? 15 : 0, evidence.sources & sourceBit("place") ? 10 : 0, evidence.order === void 0 ? 0 : SCORE.listed);
461
+ }
462
+ const tier = (score) => [
463
+ SCORE.tagged,
464
+ SCORE.used,
465
+ SCORE.listed
466
+ ].filter((bound) => score > bound).length;
467
+ /**
468
+ * A run's readings ranked, the default first: tagged ones, then those the
469
+ * sentences use, each by JMdict's order; then the lists' and the place
470
+ * names', then the other JMdict words', each by score. Readings of the same
471
+ * kana are one, under the alignment that splits them most: 倉本 くら・もと
472
+ * rather than くらもと.
473
+ */
474
+ function rankReadings(readings) {
475
+ const byKana = /* @__PURE__ */ new Map();
476
+ for (const [key, evidence] of readings) {
477
+ const spans = parseFurigana(key);
478
+ const reading = {
479
+ key,
480
+ spans,
481
+ score: runScore(evidence),
482
+ sources: sourcesOf(evidence),
483
+ order: evidence.order ?? Infinity
484
+ };
485
+ const kana = spans.map((s) => s.ruby).join("");
486
+ const same = byKana.get(kana);
487
+ if (!same) {
488
+ byKana.set(kana, reading);
489
+ continue;
490
+ }
491
+ const finer = reading.spans.length > same.spans.length ? reading : same;
492
+ byKana.set(kana, {
493
+ ...finer,
494
+ score: Math.max(same.score, reading.score),
495
+ sources: same.sources | reading.sources,
496
+ order: Math.min(same.order, reading.order)
497
+ });
498
+ }
499
+ return [...byKana.values()].sort((a, b) => tier(b.score) - tier(a.score) || (tier(a.score) > 1 ? a.order - b.order || b.score - a.score : b.score - a.score || a.order - b.order)).map((r) => [r.key, [r.score, r.sources]]);
500
+ }
501
+ /**
502
+ * When the sentences use a kanji alone often enough to tell, each reading
503
+ * scores by its share of those uses, so that its weight in the planner is
504
+ * about that share; a tagged one keeps a little, to stay among its readings.
505
+ */
506
+ function scoreUses(readings, uses) {
507
+ const total = [...uses.values()].reduce((a, b) => a + b, 0);
508
+ if (total < LONE_MIN_USES) return;
509
+ for (const reading of readings.values()) reading[0] = Math.min(reading[0], LONE_TAGGED_FLOOR);
510
+ for (const [ruby, n] of uses) {
511
+ const reading = getOrCreate(readings, ruby, () => [0, sourceBit("jmdict")]);
512
+ reading[0] = Math.max(reading[0], scoreOf(n / total));
513
+ if (n >= MIN_USES) reading[1] |= sourceBit("tatoeba");
514
+ }
515
+ }
516
+ /** A lone kanji's readings, the most common first: tagged words and the sentences score them; rare verbs add readings that score nothing. */
517
+ function kanjiReadings(kanji) {
518
+ const out = /* @__PURE__ */ new Map();
519
+ for (const [text, tagged] of kanji.tagged) {
520
+ const readings = new Map([...tagged].map(([ruby, evidence]) => [ruby, [scoreOf(1 / evidence.rank), sourcesOf(evidence)]]));
521
+ scoreUses(readings, kanji.uses.get(text) ?? /* @__PURE__ */ new Map());
522
+ out.set(text, new Map([...readings].filter(([, [score]]) => score).sort((a, b) => b[1][0] - a[1][0])));
523
+ }
524
+ for (const [text, rubies] of kanji.inflecting) {
525
+ const readings = getOrCreate(out, text, () => /* @__PURE__ */ new Map());
526
+ for (const ruby of rubies) {
527
+ const [score, sources] = readings.get(ruby) ?? [0, sourceBit("jmdict")];
528
+ readings.set(ruby, [score, sources | sourceBit("inflecting")]);
529
+ }
530
+ }
531
+ return out;
532
+ }
533
+ function notationRuns$1(notation) {
534
+ const { form, spans } = parseNotation(notation);
535
+ return hanRuns(form, spans);
536
+ }
537
+ /** The runs each list names, as the planner reads them: no lone kanji, which would be no evidence for the kanji's reading. */
538
+ function listedRuns(sources) {
539
+ const manual = MANUAL_SOURCES.flatMap((name) => (sources.manual[name] ?? []).flatMap(notationRuns$1).map((run) => [run, name]));
540
+ const places = sources.places.flatMap((place) => {
541
+ const runs = hanRuns(place.form, parseFurigana(place.furigana));
542
+ return runs.length === 1 && runs[0].text === place.form ? [[runs[0], "place"]] : [];
543
+ });
544
+ return [...manual, ...places].filter(([run]) => [...run.text].length > 1);
545
+ }
546
+ function addSource(readings, key, source) {
547
+ const [score, sources] = readings.get(key) ?? [0, 0];
548
+ readings.set(key, [score, sources | sourceBit(source)]);
549
+ }
550
+ /**
551
+ * Merges every source into the readings of the Han runs a font sees, each
552
+ * reading with its score and sources. What no plan can use is left out: a
553
+ * word's runs with an unannotated kanji, lone kanji of the lists or of words
554
+ * no list tags but before okurigana, and the on'yomi of kanji no reading uses.
555
+ */
556
+ function fuseDictionary(sources) {
557
+ const runs = /* @__PURE__ */ new Map();
558
+ const kanji = {
559
+ tagged: /* @__PURE__ */ new Map(),
560
+ uses: /* @__PURE__ */ new Map(),
561
+ inflecting: /* @__PURE__ */ new Map()
562
+ };
563
+ for (const entry of sources.entries) for (const run of hanRuns(entry.form, parseFurigana(entry.furigana))) if ([...run.text].length > 1) addRun(runs, run, entry);
564
+ else addKanji(kanji, run, entry);
565
+ for (const [run, name] of listedRuns(sources)) evidenceOf(runs, run.text, formatFurigana(run.spans)).sources |= sourceBit(name);
566
+ const ranked = new Map([...runs].map(([text, readings]) => [text, new Map(rankReadings(readings))]));
567
+ const lone = kanjiReadings(kanji);
568
+ for (const run of sources.defaults.flatMap(notationRuns$1)) {
569
+ const [table, key] = [...run.text].length > 1 ? [ranked, formatFurigana(run.spans)] : [lone, run.spans[0].ruby];
570
+ addSource(getOrCreate(table, run.text, () => /* @__PURE__ */ new Map()), key, "default");
571
+ }
572
+ const used = new Set([...ranked.keys(), ...lone.keys()].flatMap((text) => [...text]));
573
+ const toRecord = (table) => Object.fromEntries([...table].map(([text, readings]) => [text, Object.fromEntries(readings)]));
574
+ return {
575
+ versions: sources.versions,
576
+ sources: SOURCE_BITS,
577
+ onyomi: Object.fromEntries(Object.entries(sources.onyomi).filter(([k]) => used.has(k))),
578
+ runs: toRecord(ranked),
579
+ kanji: toRecord(lone),
580
+ bare: sources.bare
581
+ };
582
+ }
182
583
  //#endregion
183
584
  //#region src/glyphs.ts
184
585
  function glyphName(codepoint) {
@@ -529,94 +930,67 @@ const MIN_ALTERNATE_SHARE = .02;
529
930
  /** A suffix reading is the one a kanji takes in this share of the runs it ends, out of at least `SUFFIX_MIN_RUNS`. */
530
931
  const SUFFIX_MIN_SHARE = .9;
531
932
  const SUFFIX_MIN_RUNS = 10;
532
- /** With this many uses alone in the example sentences, they decide a lone kanji rather than the word lists. */
533
- const LONE_MIN_USES = 20;
534
- /** JMdict orders readings within a form; a run read inside a longer form ranks after the form's own readings. */
535
- const OTHER_FORM_ORDER = 1e3;
536
933
  const length = (text) => [...text].length;
537
934
  function spanKey(spans) {
538
935
  return spans.map((s) => `${s.start}-${s.end}:${s.ruby}`).join(";");
539
936
  }
937
+ /** Pinned first; then readings that are evidence, before a place name's or a rare word's; then as the dictionary ranks them. */
540
938
  function rank(a, b) {
541
- return (b.pinned ?? 0) - (a.pinned ?? 0) || Number(b.tagged) - Number(a.tagged) || a.order - b.order || b.weight - a.weight;
939
+ return (b.pinned ?? 0) - (a.pinned ?? 0) || Number(a.place ?? false) - Number(b.place ?? false) || a.order - b.order;
542
940
  }
543
- /** Rough corpus frequency of an entry: nf buckets hold ~500 words each, ranked by frequency. */
544
- function entryWeight(entry) {
545
- if (!entry.priority) return 0;
546
- const nf = entry.nf || Number(/nf(\d+)/.exec(entry.priority)?.[1] ?? 0);
547
- if (nf) return 1 / nf;
548
- return /[a-z]1\b/.test(entry.priority) ? 1 / 24 : 1 / 48;
549
- }
550
- function keepBest(current, value) {
551
- current.weight = Math.max(current.weight, value.weight);
552
- if (value.nf && (!current.nf || value.nf < current.nf)) current.nf = value.nf;
553
- current.jlpt = Math.max(current.jlpt, value.jlpt);
554
- for (const tag of value.tags) current.tags.add(tag);
555
- }
556
- function priorityTags(entry) {
557
- return new Set(entry.priority.split(" ").filter(Boolean));
558
- }
559
- function getOrCreate(map, key, create) {
560
- let value = map.get(key);
561
- if (value === void 0) map.set(key, value = create());
562
- return value;
563
- }
564
- function indexRun(index, run, entry) {
565
- const order = run.text === entry.form ? entry.order : OTHER_FORM_ORDER + entry.order;
566
- const reading = {
567
- spans: run.spans,
568
- weight: entryWeight(entry),
569
- tagged: !!entry.priority,
570
- order,
571
- nf: entry.priority ? entry.nf : 0,
572
- jlpt: entry.jlpt ?? 0,
573
- tags: priorityTags(entry),
574
- tatoeba: entry.tatoeba ?? 0
575
- };
576
- const readings = getOrCreate(index.runs, run.text, () => /* @__PURE__ */ new Map());
577
- const current = readings.get(spanKey(run.spans));
578
- if (!current) {
579
- readings.set(spanKey(run.spans), reading);
580
- return;
941
+ const isJmdict = (reading) => !!(reading.sources & sourceBit("jmdict"));
942
+ const PIN = sourceBit("default");
943
+ function indexRuns(index, runs) {
944
+ for (const [text, readings] of Object.entries(runs)) {
945
+ const ranked = [];
946
+ for (const [furigana, [score, sources]] of Object.entries(readings)) {
947
+ const spans = parseFurigana(furigana);
948
+ if (sources & PIN) index.defaults.push({
949
+ text,
950
+ spans
951
+ });
952
+ if (sources & ~PIN) ranked.push({
953
+ spans,
954
+ weight: weightOf(score),
955
+ score,
956
+ sources,
957
+ order: ranked.length
958
+ });
959
+ }
960
+ index.runs.set(text, ranked);
961
+ if (ranked.some(isJmdict)) index.jmdict.set(text, ranked.filter(isJmdict));
581
962
  }
582
- keepBest(current, reading);
583
- current.tagged ||= reading.tagged;
584
- current.order = Math.min(current.order, reading.order);
585
- current.tatoeba = Math.max(current.tatoeba ?? 0, reading.tatoeba);
586
963
  }
587
- function indexSingle(index, run, entry) {
588
- const ruby = run.spans[0].ruby;
589
- const readings = getOrCreate(index.singles, run.text, () => /* @__PURE__ */ new Map());
590
- for (const r of entry.conjugatesReading && ruby === "く" ? [
591
- "く",
592
- "き",
593
- "こ"
594
- ] : [ruby]) {
595
- const value = {
596
- weight: entryWeight(entry),
597
- nf: entry.nf,
598
- jlpt: entry.jlpt ?? 0,
599
- tags: priorityTags(entry)
600
- };
601
- const current = readings.get(r);
602
- if (current) keepBest(current, value);
603
- else readings.set(r, value);
964
+ function indexKanji(index, kanji) {
965
+ for (const [text, readings] of Object.entries(kanji)) for (const [ruby, [score, sources]] of Object.entries(readings)) {
966
+ if (weightOf(score)) getOrCreate(index.singles, text, () => /* @__PURE__ */ new Map()).set(ruby, {
967
+ weight: weightOf(score),
968
+ score,
969
+ sources
970
+ });
971
+ if (sources & sourceBit("inflecting")) getOrCreate(index.inflecting, text, () => /* @__PURE__ */ new Set()).add(ruby);
972
+ if (sources & PIN) index.defaults.push({
973
+ text,
974
+ spans: [{
975
+ start: 0,
976
+ end: 1,
977
+ ruby
978
+ }]
979
+ });
604
980
  }
605
981
  }
606
- const OKURIGANA = /[\u3041-\u3096]/;
607
- function indexEntry(index, entry) {
608
- for (const run of hanRuns(entry.form, parseFurigana(entry.furigana))) {
609
- if (length(run.text) > 1) {
610
- indexRun(index, run, entry);
611
- continue;
612
- }
613
- if (entry.tatoeba && !entry.conjugatesReading) {
614
- const uses = getOrCreate(index.loneUses, run.text, () => /* @__PURE__ */ new Map());
615
- uses.set(run.spans[0].ruby, (uses.get(run.spans[0].ruby) ?? 0) + entry.tatoeba);
616
- }
617
- if (entry.priority) indexSingle(index, run, entry);
618
- else if (OKURIGANA.test(entry.form[entry.form.indexOf(run.text) + run.text.length] ?? "")) getOrCreate(index.inflecting, run.text, () => /* @__PURE__ */ new Set()).add(run.spans[0].ruby);
619
- }
982
+ /** The compiled dictionary as the planner looks it up. */
983
+ function indexDictionary(dictionary) {
984
+ const index = {
985
+ runs: /* @__PURE__ */ new Map(),
986
+ jmdict: /* @__PURE__ */ new Map(),
987
+ singles: /* @__PURE__ */ new Map(),
988
+ inflecting: /* @__PURE__ */ new Map(),
989
+ defaults: []
990
+ };
991
+ indexRuns(index, dictionary.runs);
992
+ indexKanji(index, dictionary.kanji);
993
+ return index;
620
994
  }
621
995
  function createContext(dictionary, options, coverage) {
622
996
  const covered = (text) => codepoints(text).every((cp) => coverage.has(cp));
@@ -627,20 +1001,22 @@ function createContext(dictionary, options, coverage) {
627
1001
  if (options.rubyScript === "katakanaOn" && kanji.length === 1 && isOnyomi(span.ruby, dictionary.onyomi[kanji[0]] ?? [])) return toKatakana(span.ruby);
628
1002
  return span.ruby;
629
1003
  };
630
- const known = options.skipJlpt ? Number(options.skipJlpt.slice(1)) : Infinity;
1004
+ const known = options.skipJlpt ? JLPT_LEVELS.slice(0, JLPT_LEVELS.indexOf(options.skipJlpt) + 1).reduce((mask, level) => mask | sourceBit(level), 0) : 0;
631
1005
  return {
632
1006
  options,
633
- removed: /* @__PURE__ */ new Set([...dictionary.bare ?? [], ...options.words.remove]),
1007
+ removed: /* @__PURE__ */ new Set([...dictionary.bare, ...options.words.remove]),
634
1008
  covered,
635
- selected: (r) => r.nf >= options.nfMin && r.nf <= options.nfMax || options.selectTags.some((t) => r.tags.has(t)) || options.minTatoeba > 0 && (r.tatoeba ?? 0) >= options.minTatoeba,
636
- skipped: (r) => r.nf > 0 && r.nf < options.nfMin || r.jlpt >= known,
1009
+ known: (sources) => !!(sources & known),
637
1010
  render,
638
1011
  renderable: (run) => covered(run.text) && run.spans.every((_, i) => covered(render(run, i)))
639
1012
  };
640
1013
  }
641
- function addCustomWord(patterns, lonePins, notation, pinned, ctx) {
1014
+ function notationRuns(notation) {
642
1015
  const { form, spans } = parseNotation(notation);
643
- for (const run of hanRuns(form, spans).filter(ctx.renderable)) {
1016
+ return hanRuns(form, spans);
1017
+ }
1018
+ function addCustomRuns(patterns, lonePins, runs, pinned, ctx) {
1019
+ for (const run of runs.filter(ctx.renderable)) {
644
1020
  if (length(run.text) === 1) {
645
1021
  if (pinned) lonePins.set(run.text, run.spans[0].ruby);
646
1022
  continue;
@@ -652,101 +1028,69 @@ function addCustomWord(patterns, lonePins, notation, pinned, ctx) {
652
1028
  else readings.push({
653
1029
  spans: run.spans,
654
1030
  weight: 1,
655
- tagged: true,
1031
+ score: 100,
1032
+ sources: 0,
656
1033
  order: Number.MAX_SAFE_INTEGER,
657
- nf: 1,
658
- jlpt: 0,
659
- tags: /* @__PURE__ */ new Set(),
660
1034
  pinned
661
1035
  });
662
1036
  }
663
1037
  }
664
- /** Place names fill the runs no common word claims, nor a skipped one. */
665
- function addPlaces(patterns, places, skipped, ctx) {
666
- for (const place of places ?? []) {
667
- const spans = parseFurigana(place.furigana);
668
- if (patterns.has(place.form) || skipped.has(place.form) || ctx.removed.has(place.form) || !ctx.renderable({
669
- text: place.form,
670
- spans
671
- })) continue;
672
- patterns.set(place.form, [{
673
- spans,
674
- weight: 0,
675
- tagged: false,
676
- order: 0,
677
- nf: 0,
678
- jlpt: 0,
679
- tags: /* @__PURE__ */ new Set(),
680
- place: true
681
- }]);
682
- }
683
- }
684
- /** Words of the manual sources fill the runs no common word claims, nor a skipped one; they come before place names. */
685
- function addManual(patterns, manual, skipped, ctx) {
686
- const names = ctx.options.manual ?? Object.keys(manual);
687
- for (const notation of names.flatMap((name) => manual[name] ?? [])) {
688
- const { form, spans } = parseNotation(notation);
689
- for (const run of hanRuns(form, spans).filter(ctx.renderable)) {
690
- if (length(run.text) === 1 || patterns.has(run.text) || skipped.has(run.text) || ctx.removed.has(run.text)) continue;
691
- patterns.set(run.text, [{
692
- spans: run.spans,
693
- weight: 0,
694
- tagged: false,
695
- order: 0,
696
- nf: 0,
697
- jlpt: 0,
698
- tags: /* @__PURE__ */ new Set(),
699
- place: true
700
- }]);
701
- }
702
- }
703
- }
704
- /** Words no list ranks fill the runs nothing else claims, each with its first reading: they are no evidence for the kanji's own. */
705
- function addUntagged(patterns, index, skipped, ctx) {
706
- for (const [text, readings] of index.runs) {
707
- if (patterns.has(text) || skipped.has(text) || ctx.removed.has(text)) continue;
708
- const first = [...readings.values()].filter((r) => ctx.renderable({
709
- text,
710
- spans: r.spans
711
- })).sort((a, b) => a.order - b.order)[0];
712
- if (first) patterns.set(text, [{
713
- ...first,
714
- place: true
715
- }]);
716
- }
1038
+ /** A run's readings from the plan's sources that the font can draw: those the lists or the sentences back, else the first, which is no evidence for its kanji's. */
1039
+ function wordReadings(text, readings, wanted, ctx) {
1040
+ const selected = readings.filter((r) => r.sources & wanted && ctx.renderable({
1041
+ text,
1042
+ spans: r.spans
1043
+ }));
1044
+ const backed = selected.filter((r) => r.score > SCORE.used);
1045
+ return backed.length ? backed.map((r) => ({ ...r })) : selected.slice(0, 1).map((r) => ({
1046
+ ...r,
1047
+ place: true
1048
+ }));
717
1049
  }
718
1050
  /**
719
- * Runs of common words, then place names and custom words; pinned readings of
720
- * lone kanji are returned apart. A word is skipped whole: were only its known
721
- * reading dropped, 今日 would show こんにち.
1051
+ * The words of the plan's sources, but those the reader knows, those the
1052
+ * user removes and, by score, the `skip` most common, words and lone kanji
1053
+ * together; then the `take` next words, and the user's own. A word is skipped
1054
+ * whole: were only its known reading dropped, 今日 would show こんにち.
722
1055
  */
723
- function selectWords(index, dictionary, ctx) {
724
- const patterns = /* @__PURE__ */ new Map();
1056
+ function selectWords(index, ctx) {
1057
+ const wanted = ctx.options.sources.reduce((mask, source) => mask | sourceBit(source), 0);
725
1058
  const skipped = /* @__PURE__ */ new Set();
1059
+ const skippedKanji = /* @__PURE__ */ new Set();
1060
+ const words = [];
726
1061
  for (const [text, readings] of index.runs) {
727
1062
  if (ctx.removed.has(text)) continue;
728
- if ([...readings.values()].some(ctx.skipped)) {
1063
+ if (readings.some((r) => ctx.known(r.sources))) {
729
1064
  skipped.add(text);
730
1065
  continue;
731
1066
  }
732
- const common = [...readings.values()].filter((r) => ctx.selected(r) && ctx.renderable({
733
- text,
734
- spans: r.spans
735
- }));
736
- if (common.length) patterns.set(text, common.map((r) => ({ ...r })));
1067
+ const chosen = wordReadings(text, readings, wanted, ctx);
1068
+ if (chosen.length) words.push([text, chosen]);
737
1069
  }
738
- if (dictionary.manual) addManual(patterns, dictionary.manual, skipped, ctx);
739
- if (ctx.options.places) addPlaces(patterns, dictionary.places, skipped, ctx);
740
- if (ctx.options.untagged) addUntagged(patterns, index, skipped, ctx);
1070
+ const kanji = [...index.singles].map(([text, readings]) => [text, [...readings.values()].reduce((a, b) => b.score > a.score ? b : a)]);
1071
+ for (const [text] of kanji.filter(([, top]) => ctx.known(top.sources))) skippedKanji.add(text);
1072
+ const byScore = [...words.map(([text, readings]) => ({
1073
+ text,
1074
+ score: Math.max(...readings.map((r) => r.score)),
1075
+ word: true
1076
+ })), ...kanji.filter(([text]) => !skippedKanji.has(text)).map(([text, top]) => ({
1077
+ text,
1078
+ score: top.score,
1079
+ word: false
1080
+ }))].sort((a, b) => b.score - a.score);
1081
+ for (const { text, word } of byScore.slice(0, ctx.options.skip)) (word ? skipped : skippedKanji).add(text);
1082
+ const taken = new Set(byScore.slice(ctx.options.skip).filter((item) => item.word).slice(0, ctx.options.take).map((item) => item.text));
1083
+ const patterns = new Map(words.filter(([text]) => taken.has(text)));
741
1084
  const lonePins = /* @__PURE__ */ new Map();
742
- for (const notation of ctx.options.words.add) addCustomWord(patterns, lonePins, notation, 0, ctx);
743
- for (const notation of dictionary.defaults ?? []) addCustomWord(patterns, lonePins, notation, 1, ctx);
744
- for (const notation of ctx.options.words.pin) addCustomWord(patterns, lonePins, notation, 2, ctx);
1085
+ addCustomRuns(patterns, lonePins, ctx.options.words.add.flatMap(notationRuns), 0, ctx);
1086
+ addCustomRuns(patterns, lonePins, index.defaults, 1, ctx);
1087
+ addCustomRuns(patterns, lonePins, ctx.options.words.pin.flatMap(notationRuns), 2, ctx);
745
1088
  for (const readings of patterns.values()) readings.sort(rank).splice(15);
746
1089
  return {
747
1090
  patterns,
748
1091
  lonePins,
749
- skipped
1092
+ skipped,
1093
+ skippedKanji
750
1094
  };
751
1095
  }
752
1096
  /** Readings of a lone kanji the font can draw, heaviest first, with their share of the evidence. */
@@ -766,24 +1110,6 @@ function loneReadings(kanji, readings, ctx) {
766
1110
  share: r.weight / total
767
1111
  })).sort((a, b) => b.weight - a.weight);
768
1112
  }
769
- /** The reading running text gives a lone kanji most, with its share of the uses, when the sentences use the kanji alone often enough to tell. */
770
- function usedReading(kanji, uses, ctx) {
771
- const drawable = [...uses ?? []].filter(([ruby]) => ctx.renderable({
772
- text: kanji,
773
- spans: [{
774
- start: 0,
775
- end: 1,
776
- ruby
777
- }]
778
- }));
779
- const total = drawable.reduce((sum, [, n]) => sum + n, 0);
780
- if (ctx.options.minTatoeba === 0 || total < LONE_MIN_USES) return;
781
- const [ruby, n] = drawable.sort((a, b) => b[1] - a[1])[0];
782
- return {
783
- ruby,
784
- share: n / total
785
- };
786
- }
787
1113
  function addWordEvidence(patterns, add) {
788
1114
  for (const [text, readings] of patterns) {
789
1115
  const chars = [...text];
@@ -803,12 +1129,13 @@ function addRareVerbEvidence(inflecting, ctx, add) {
803
1129
  }
804
1130
  /**
805
1131
  * A kanji alone in its run gets ruby only when one reading clearly dominates
806
- * the common words it stands alone in, whatever the word list's size: verbs
1132
+ * the common words it stands alone in, or the sentences' uses of it alone
1133
+ * (ゴミ箱 is common, yet 箱 reads はこ), whatever the word list's size: verbs
807
1134
  * like 書く rank low in newspapers yet decide how 書 reads. Every kanji also
808
1135
  * gets an ordered list of its readings, which the variation selectors index
809
1136
  * into, except a kanji whose dominant reading the reader is taken to know.
810
1137
  */
811
- function chooseLoneKanji(index, patterns, lonePins, ctx) {
1138
+ function chooseLoneKanji(index, { patterns, lonePins, skippedKanji }, ctx) {
812
1139
  const isolated = new Map(ctx.options.isolated ? lonePins : []);
813
1140
  const skipped = /* @__PURE__ */ new Set();
814
1141
  const evidence = /* @__PURE__ */ new Map();
@@ -821,9 +1148,8 @@ function chooseLoneKanji(index, patterns, lonePins, ctx) {
821
1148
  if (ctx.removed.has(kanji)) continue;
822
1149
  const ranked = loneReadings(kanji, readings, ctx).filter((r) => r.share >= MIN_ALTERNATE_SHARE);
823
1150
  const top = ranked[0];
824
- const chosen = usedReading(kanji, index.loneUses.get(kanji), ctx) ?? top;
825
- if (top && !lonePins.has(kanji) && ctx.skipped(readings.get(top.ruby))) skipped.add(kanji);
826
- else if (chosen && chosen.share >= ctx.options.isolatedMinShare && ctx.options.isolated && !isolated.has(kanji)) isolated.set(kanji, chosen.ruby);
1151
+ if (top && !lonePins.has(kanji) && skippedKanji.has(kanji)) skipped.add(kanji);
1152
+ else if (top && top.share >= ctx.options.isolatedMinShare && ctx.options.isolated && !isolated.has(kanji)) isolated.set(kanji, top.ruby);
827
1153
  for (const r of ranked) add(kanji, r.ruby, r.weight);
828
1154
  }
829
1155
  addWordEvidence(patterns, add);
@@ -854,11 +1180,11 @@ function chooseSuffixes(index, ctx) {
854
1180
  const count = (counts, key) => void counts.set(key, (counts.get(key) ?? 0) + 1);
855
1181
  const starts = /* @__PURE__ */ new Map();
856
1182
  const ends = /* @__PURE__ */ new Map();
857
- for (const [text, readings] of index.runs) {
1183
+ for (const [text, readings] of index.jmdict) {
858
1184
  const chars = [...text];
859
1185
  count(starts, chars[0]);
860
1186
  count(ends, chars.at(-1));
861
- for (const { spans } of readings.values()) {
1187
+ for (const { spans } of readings) {
862
1188
  const span = spans.at(-1);
863
1189
  if (span.end - span.start === 1) count(getOrCreate(evidence, chars.at(-1), () => /* @__PURE__ */ new Map()), span.ruby);
864
1190
  }
@@ -922,10 +1248,10 @@ function findMisreadRuns(index, patterns, kanjiReadings, suffixes, skipped, ctx)
922
1248
  }
923
1249
  const protect = (text, readings, longest) => {
924
1250
  const chars = [...text];
925
- const candidates = [...new Set([...readings.values()].map((r) => assignment(r.spans, chars.length).join("|")))];
926
- const expected = [...readings.values()].map((r) => assignment(r.spans, chars.length));
1251
+ const candidates = [...new Set(readings.map((r) => assignment(r.spans, chars.length).join("|")))];
1252
+ const expected = readings.map((r) => assignment(r.spans, chars.length));
927
1253
  if (!expected.some((e) => misreads(chars, e, known, longest, suffixes))) return false;
928
- const truth = [...readings.values()].sort(rank)[0];
1254
+ const truth = readings.toSorted(rank)[0];
929
1255
  const run = {
930
1256
  text,
931
1257
  spans: truth.spans
@@ -939,7 +1265,7 @@ function findMisreadRuns(index, patterns, kanjiReadings, suffixes, skipped, ctx)
939
1265
  let longest = 2;
940
1266
  for (const text of known.keys()) longest = Math.max(longest, length(text));
941
1267
  changed = false;
942
- for (const [text, readings] of index.runs) if (!known.has(text) && ctx.covered(text) && protect(text, readings, longest)) changed = true;
1268
+ for (const [text, readings] of index.jmdict) if (!known.has(text) && ctx.covered(text) && protect(text, readings, longest)) changed = true;
943
1269
  }
944
1270
  return protectedRuns;
945
1271
  }
@@ -1064,7 +1390,7 @@ function buildRules(patterns, kanjiReadings, protectedRuns, glyphs) {
1064
1390
  };
1065
1391
  }
1066
1392
  function smokeChecks(patterns, lone, glyphs) {
1067
- const byFrequency = [...patterns].sort((a, b) => (a[1][0].nf || 99) - (b[1][0].nf || 99));
1393
+ const byFrequency = [...patterns].sort((a, b) => b[1][0].score - a[1][0].score);
1068
1394
  const checks = [];
1069
1395
  for (const grouped of [false, true]) for (const [text, readings] of byFrequency.filter(([text, r]) => isGrouped({
1070
1396
  text,
@@ -1094,24 +1420,19 @@ function smokeChecks(patterns, lone, glyphs) {
1094
1420
  return checks;
1095
1421
  }
1096
1422
  /**
1097
- * Indexes the dictionary once into Han runs, then plans fonts from it. Every
1423
+ * Indexes the compiled dictionary once, then plans fonts from it. Every
1098
1424
  * decision about readings happens here: which words get ruby, which reading
1099
1425
  * is the default, which longer words must be protected from a shorter match.
1100
1426
  */
1101
1427
  function createFontPlanner(dictionary) {
1102
- const index = {
1103
- runs: /* @__PURE__ */ new Map(),
1104
- singles: /* @__PURE__ */ new Map(),
1105
- inflecting: /* @__PURE__ */ new Map(),
1106
- loneUses: /* @__PURE__ */ new Map()
1107
- };
1108
- for (const entry of dictionary.entries) indexEntry(index, entry);
1109
- const sources = Object.entries(dictionary.sources).map(([name, version]) => `${name} ${version}`).join(", ");
1428
+ const index = indexDictionary(dictionary);
1429
+ const sources = Object.entries(dictionary.versions).map(([name, version]) => `${name} ${version}`).join(", ");
1110
1430
  /** Every decision for one set of options, over the whole dictionary: slow, and the same whatever the preview text. */
1111
1431
  function analyze(options, coverage) {
1112
1432
  const ctx = createContext(dictionary, options, coverage);
1113
- const { patterns, lonePins, skipped } = selectWords(index, dictionary, ctx);
1114
- const { isolated, kanjiReadings } = chooseLoneKanji(index, patterns, lonePins, ctx);
1433
+ const selection = selectWords(index, ctx);
1434
+ const { patterns, skipped } = selection;
1435
+ const { isolated, kanjiReadings } = chooseLoneKanji(index, selection, ctx);
1115
1436
  const suffixes = chooseSuffixes(index, ctx);
1116
1437
  const protectedRuns = findMisreadRuns(index, patterns, kanjiReadings, suffixes, skipped, ctx);
1117
1438
  const glyphs = createGlyphRegistry(ctx.render);
@@ -1246,148 +1567,4 @@ function createFontPlanner(dictionary) {
1246
1567
  };
1247
1568
  }
1248
1569
  //#endregion
1249
- //#region src/schema.ts
1250
- const RUBY_SCRIPTS = [
1251
- "hiragana",
1252
- "katakanaOn",
1253
- "hepburn"
1254
- ];
1255
- const PRIORITY_TAGS = [
1256
- "ichi1",
1257
- "ichi2",
1258
- "news1",
1259
- "news2",
1260
- "spec1",
1261
- "spec2",
1262
- "gai1",
1263
- "gai2"
1264
- ];
1265
- /** Easiest first. */
1266
- /** The dictionary's manually maintained sources, which a plan may include by name. */
1267
- const MANUAL_SOURCES = [
1268
- "mahjong-yaku",
1269
- "jr-stations",
1270
- "yojijukugo"
1271
- ];
1272
- const JLPT_LEVELS = [
1273
- "N5",
1274
- "N4",
1275
- "N3",
1276
- "N2",
1277
- "N1"
1278
- ];
1279
- /** Sizes in em, relative to the base font. */
1280
- const RubyMetricsSchema = v.object({
1281
- /** Ruby size relative to the base characters. */
1282
- rubyScale: v.optional(v.pipe(v.number(), v.minValue(.2), v.maxValue(.8)), DEFAULTS.plan.metrics.rubyScale),
1283
- /** Space between the top of the em box and the bottom of the ruby. */
1284
- rubyGap: v.optional(v.pipe(v.number(), v.minValue(0), v.maxValue(.5)), DEFAULTS.plan.metrics.rubyGap),
1285
- /** Ruby is squeezed horizontally down to this scale before it may overhang. */
1286
- compressionFloor: v.optional(v.pipe(v.number(), v.minValue(.3), v.maxValue(1)), DEFAULTS.plan.metrics.compressionFloor),
1287
- /** Widest a ruby may get, overhanging its base symmetrically. */
1288
- maxRubyWidth: v.optional(v.pipe(v.number(), v.minValue(1), v.maxValue(3)), DEFAULTS.plan.metrics.maxRubyWidth)
1289
- });
1290
- const Bucket = v.pipe(v.number(), v.integer(), v.minValue(1), v.maxValue(48));
1291
- /** `schema`, checking that its frequency window `nfMin..nfMax` holds a bucket; for schemas that extend the options too. */
1292
- function withNfWindow(schema) {
1293
- return v.pipe(schema, v.check((o) => o.nfMin <= o.nfMax, "nfMin must not exceed nfMax"));
1294
- }
1295
- const PlanOptionsSchema = withNfWindow(v.object({
1296
- family: v.optional(v.pipe(v.string(), v.trim(), v.nonEmpty(), v.check((name) => !/source/i.test(name), "Source is a reserved font name of the base font")), DEFAULTS.plan.family),
1297
- rubyScript: v.optional(v.picklist(RUBY_SCRIPTS), DEFAULTS.plan.rubyScript),
1298
- /**
1299
- * Words whose newspaper frequency bucket is below this stay bare: the most
1300
- * common ones, which a reader knows. Tags don't bring them back.
1301
- */
1302
- nfMin: v.optional(Bucket, DEFAULTS.plan.nfMin),
1303
- /** Words whose newspaper frequency bucket is at most this are annotated. */
1304
- nfMax: v.optional(Bucket, DEFAULTS.plan.nfMax),
1305
- /**
1306
- * JMdict priority tags that select a word whatever its frequency bucket:
1307
- * `spec1` catches words like 日本 that the newspaper ranking misses.
1308
- */
1309
- selectTags: v.optional(v.array(v.picklist(PRIORITY_TAGS)), () => [...DEFAULTS.plan.selectTags]),
1310
- /**
1311
- * Words of two or more kanji that Tatoeba's example sentences use at least
1312
- * this many times are annotated whatever their bucket: everyday words that
1313
- * no list tags, such as 留守中, which would otherwise be bare or read as a
1314
- * place name (国中 くになか, not くにじゅう). The sentences also decide how a
1315
- * lone kanji reads, where they use it alone often enough. 0 turns them off.
1316
- */
1317
- minTatoeba: v.optional(v.pipe(v.number(), v.integer(), v.minValue(0)), DEFAULTS.plan.minTatoeba),
1318
- /** Words on the JLPT lists of this level and every easier one stay bare. */
1319
- skipJlpt: v.optional(v.picklist(JLPT_LEVELS)),
1320
- /** Annotate place names from the dictionary where no common word claims the run. */
1321
- places: v.optional(v.boolean(), DEFAULTS.plan.places),
1322
- /** Annotate the words of JMdict that no list ranks or tags (such as 四字熟語), where no common word, manual list or place name claims the run; each gets its first reading. */
1323
- untagged: v.optional(v.boolean(), DEFAULTS.plan.untagged),
1324
- /** The dictionary's manually maintained sources (such as `mahjong-yaku`) to annotate; all of them when left out. */
1325
- manual: v.optional(v.array(v.picklist(MANUAL_SOURCES))),
1326
- /** Annotate a kanji alone in its run, or one no word claims after another kanji (a suffix); when off, only selectors give them ruby. */
1327
- isolated: v.optional(v.boolean(), DEFAULTS.plan.isolated),
1328
- /** A kanji alone in its run is annotated only when one reading has this share of the evidence. */
1329
- isolatedMinShare: v.optional(v.pipe(v.number(), v.minValue(.5), v.maxValue(1)), DEFAULTS.plan.isolatedMinShare),
1330
- words: v.optional(v.object({
1331
- /** Extra words in inline notation, `日[に]本[ほん]`. */
1332
- add: v.optional(v.array(v.string()), []),
1333
- /** Written forms never to annotate. */
1334
- remove: v.optional(v.array(v.string()), []),
1335
- /** Readings to make the default, in inline notation. */
1336
- pin: v.optional(v.array(v.string()), [])
1337
- }), {}),
1338
- metrics: v.optional(RubyMetricsSchema, {})
1339
- }));
1340
- function resolveOptions(input = {}) {
1341
- return v.parse(PlanOptionsSchema, input);
1342
- }
1343
- const Codepoint = v.pipe(v.number(), v.integer(), v.minValue(0), v.maxValue(1114111));
1344
- /**
1345
- * The contract between the planner and the compiler. The compiler makes no
1346
- * decision about readings: it draws what `glyphs` describes and compiles `fea`.
1347
- * In `fea`, base characters are named `uniXXXX` / `uXXXXX` after their codepoint.
1348
- */
1349
- const PlanSchema = v.object({
1350
- version: v.literal(1),
1351
- names: v.object({
1352
- family: v.string(),
1353
- description: v.string()
1354
- }),
1355
- metrics: v.object({
1356
- rubyScale: v.number(),
1357
- rubyGap: v.number(),
1358
- compressionFloor: v.number(),
1359
- maxRubyWidth: v.number()
1360
- }),
1361
- /** Base characters the rules name; each must be in the base font's cmap. */
1362
- codepoints: v.array(Codepoint),
1363
- /**
1364
- * Base + variation selector sequences to map in `cmap` format 14, each to an
1365
- * empty glyph named `name`, with the base's advance: the rules replace it.
1366
- */
1367
- variants: v.array(v.object({
1368
- name: v.pipe(v.string(), v.regex(/^[A-Z_][\w.]{0,62}$/i)),
1369
- base: Codepoint,
1370
- selector: Codepoint
1371
- })),
1372
- glyphs: v.array(v.object({
1373
- name: v.pipe(v.string(), v.regex(/^[A-Z_][\w.]{0,62}$/i)),
1374
- /** The base characters drawn side by side; more than one makes a ligature. */
1375
- bases: v.pipe(v.array(Codepoint), v.minLength(1)),
1376
- /** Ruby text drawn above characters `[start, end)` of `bases`. */
1377
- ruby: v.array(v.object({
1378
- start: v.number(),
1379
- end: v.number(),
1380
- text: v.pipe(v.string(), v.nonEmpty())
1381
- }))
1382
- })),
1383
- fea: v.string(),
1384
- /** Subset the base font to these codepoints before building (live previews). */
1385
- subset: v.optional(v.array(Codepoint)),
1386
- /** Shaping checks the compiler runs on its output: `text` must shape to `glyphs`. */
1387
- smoke: v.array(v.object({
1388
- text: v.string(),
1389
- glyphs: v.array(v.string())
1390
- }))
1391
- });
1392
- //#endregion
1393
- export { DEFAULTS, JLPT_LEVELS, MANUAL_SOURCES, MAX_READINGS, PREBUILT, PRIORITY_TAGS, PlanOptionsSchema, PlanSchema, RUBY_SCRIPTS, SELECTOR_FIRST_READING, SELECTOR_SUPPRESS, VERSION_PROBE, createFontPlanner, formatNotation, glyphName, hanRuns, isHan, isOnyomi, parseFurigana, parseNotation, resolveOptions, toHepburn, toHiragana, toKatakana, withNfWindow };
1570
+ export { DEFAULTS, DictionarySchema, JLPT_LEVELS, MANUAL_SOURCES, MAX_READINGS, PREBUILT, PlanOptionsSchema, PlanSchema, RUBY_SCRIPTS, SCORE, SELECTOR_FIRST_READING, SELECTOR_SUPPRESS, SOURCES, VERSION_PROBE, WORD_SOURCES, createFontPlanner, formatDictionary, formatFurigana, formatNotation, fuseDictionary, glyphName, hanRuns, isHan, isOnyomi, parseFurigana, parseNotation, resolveOptions, scoreOf, sourceBit, sourcesIn, toHepburn, toHiragana, toKatakana, weightOf };