@furifont/core 0.3.0 → 0.4.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.d.mts +159 -140
- package/dist/index.mjs +546 -369
- package/package.json +1 -1
package/dist/index.mjs
CHANGED
|
@@ -10,20 +10,8 @@ const DEFAULTS = {
|
|
|
10
10
|
plan: {
|
|
11
11
|
family: "Furifont Sans",
|
|
12
12
|
rubyScript: "hiragana",
|
|
13
|
-
/**
|
|
14
|
-
|
|
15
|
-
nfMax: 48,
|
|
16
|
-
/** JMdict priority tags that select a word whatever its bucket: the first tier of each list, which JMdict marks as common. */
|
|
17
|
-
selectTags: [
|
|
18
|
-
"ichi1",
|
|
19
|
-
"news1",
|
|
20
|
-
"spec1",
|
|
21
|
-
"gai1"
|
|
22
|
-
],
|
|
23
|
-
/** Uses in Tatoeba's example sentences that select a word whatever its bucket: about 1,000 more words, and 50 glyphs. */
|
|
24
|
-
minTatoeba: 3,
|
|
25
|
-
places: true,
|
|
26
|
-
untagged: true,
|
|
13
|
+
/** No word is left bare for being common; every word of every source gets ruby. */
|
|
14
|
+
skip: 0,
|
|
27
15
|
isolated: true,
|
|
28
16
|
isolatedMinShare: .8,
|
|
29
17
|
/** Sizes in em, relative to the base font. */
|
|
@@ -82,8 +70,8 @@ const PREBUILT = {
|
|
|
82
70
|
style: "VF",
|
|
83
71
|
weights: [250, 900]
|
|
84
72
|
},
|
|
85
|
-
/** Options set on every prebuilt font over the defaults above, such as `{
|
|
86
|
-
options: {}
|
|
73
|
+
/** Options set on every prebuilt font over the defaults above, such as `{ take: 20000 }`. */
|
|
74
|
+
options: { take: 4e4 }
|
|
87
75
|
};
|
|
88
76
|
/**
|
|
89
77
|
* The sequence, 。 and VS1, a page measures to tell which version of Furifont
|
|
@@ -101,16 +89,22 @@ const HAN = /\p{Script=Han}/u;
|
|
|
101
89
|
function isHan(char) {
|
|
102
90
|
return HAN.test(char);
|
|
103
91
|
}
|
|
92
|
+
/** Spans of a JmdictFurigana alignment; a loop, not splits, as the planner parses the 400,000 readings of the dictionary as it starts. */
|
|
104
93
|
function parseFurigana(furigana) {
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
94
|
+
const spans = [];
|
|
95
|
+
for (const part of furigana.split(";")) {
|
|
96
|
+
if (!part) continue;
|
|
97
|
+
const colon = part.indexOf(":");
|
|
98
|
+
const range = colon < 0 ? part : part.slice(0, colon);
|
|
99
|
+
const dash = range.indexOf("-");
|
|
100
|
+
const start = Number(dash < 0 ? range : range.slice(0, dash));
|
|
101
|
+
spans.push({
|
|
109
102
|
start,
|
|
110
|
-
end:
|
|
111
|
-
ruby
|
|
112
|
-
};
|
|
113
|
-
}
|
|
103
|
+
end: (dash < 0 ? start : Number(range.slice(dash + 1))) + 1,
|
|
104
|
+
ruby: colon < 0 ? "" : part.slice(colon + 1)
|
|
105
|
+
});
|
|
106
|
+
}
|
|
107
|
+
return spans;
|
|
114
108
|
}
|
|
115
109
|
/**
|
|
116
110
|
* Splits a form into its Han runs. Browsers itemize text by script before
|
|
@@ -179,6 +173,413 @@ function formatNotation(text, spans) {
|
|
|
179
173
|
}
|
|
180
174
|
return out + chars.slice(at).join("");
|
|
181
175
|
}
|
|
176
|
+
function getOrCreate(map, key, create) {
|
|
177
|
+
let value = map.get(key);
|
|
178
|
+
if (value === void 0) map.set(key, value = create());
|
|
179
|
+
return value;
|
|
180
|
+
}
|
|
181
|
+
//#endregion
|
|
182
|
+
//#region src/schema.ts
|
|
183
|
+
const RUBY_SCRIPTS = [
|
|
184
|
+
"hiragana",
|
|
185
|
+
"katakanaOn",
|
|
186
|
+
"hepburn"
|
|
187
|
+
];
|
|
188
|
+
/** The dictionary's manually maintained sources. */
|
|
189
|
+
const MANUAL_SOURCES = [
|
|
190
|
+
"mahjong-yaku",
|
|
191
|
+
"jr-stations",
|
|
192
|
+
"yojijukugo",
|
|
193
|
+
"japan-post"
|
|
194
|
+
];
|
|
195
|
+
/**
|
|
196
|
+
* The sources a plan takes words from: any JMdict word; JMdict's priority
|
|
197
|
+
* lists (both tiers of each) and its newspaper ranking; words Tatoeba's
|
|
198
|
+
* sentences use at least three times; JMnedict's place names; the manual lists.
|
|
199
|
+
*/
|
|
200
|
+
const WORD_SOURCES = [
|
|
201
|
+
"jmdict",
|
|
202
|
+
"ichi",
|
|
203
|
+
"news",
|
|
204
|
+
"spec",
|
|
205
|
+
"gai",
|
|
206
|
+
"nf",
|
|
207
|
+
"tatoeba",
|
|
208
|
+
"place",
|
|
209
|
+
...MANUAL_SOURCES
|
|
210
|
+
];
|
|
211
|
+
/** Easiest first. */
|
|
212
|
+
const JLPT_LEVELS = [
|
|
213
|
+
"N5",
|
|
214
|
+
"N4",
|
|
215
|
+
"N3",
|
|
216
|
+
"N2",
|
|
217
|
+
"N1"
|
|
218
|
+
];
|
|
219
|
+
/**
|
|
220
|
+
* Every source a reading may have, a bit each in this order: the word
|
|
221
|
+
* sources, then the easiest JLPT level listing it, `inflecting` (a kanji read
|
|
222
|
+
* before okurigana in a JMdict word no list tags, such as 奉る) and `default`,
|
|
223
|
+
* the dictionary's curated default.
|
|
224
|
+
*/
|
|
225
|
+
const SOURCES = [
|
|
226
|
+
...WORD_SOURCES,
|
|
227
|
+
...JLPT_LEVELS,
|
|
228
|
+
"inflecting",
|
|
229
|
+
"default"
|
|
230
|
+
];
|
|
231
|
+
const sourceBit = (source) => 1 << SOURCES.indexOf(source);
|
|
232
|
+
/** The sources a mask of `SOURCES` bits names. */
|
|
233
|
+
const sourcesIn = (mask) => SOURCES.filter((source) => mask & sourceBit(source));
|
|
234
|
+
/** Sizes in em, relative to the base font. */
|
|
235
|
+
const RubyMetricsSchema = v.object({
|
|
236
|
+
/** Ruby size relative to the base characters. */
|
|
237
|
+
rubyScale: v.optional(v.pipe(v.number(), v.minValue(.2), v.maxValue(.8)), DEFAULTS.plan.metrics.rubyScale),
|
|
238
|
+
/** Space between the top of the em box and the bottom of the ruby. */
|
|
239
|
+
rubyGap: v.optional(v.pipe(v.number(), v.minValue(0), v.maxValue(.5)), DEFAULTS.plan.metrics.rubyGap),
|
|
240
|
+
/** Ruby is squeezed horizontally down to this scale before it may overhang. */
|
|
241
|
+
compressionFloor: v.optional(v.pipe(v.number(), v.minValue(.3), v.maxValue(1)), DEFAULTS.plan.metrics.compressionFloor),
|
|
242
|
+
/** Widest a ruby may get, overhanging its base symmetrically. */
|
|
243
|
+
maxRubyWidth: v.optional(v.pipe(v.number(), v.minValue(1), v.maxValue(3)), DEFAULTS.plan.metrics.maxRubyWidth)
|
|
244
|
+
});
|
|
245
|
+
const Count = v.pipe(v.number(), v.integer(), v.minValue(0));
|
|
246
|
+
const PlanOptionsSchema = v.object({
|
|
247
|
+
family: v.optional(v.pipe(v.string(), v.trim(), v.nonEmpty(), v.check((name) => !/source/i.test(name), "Source is a reserved font name of the base font")), DEFAULTS.plan.family),
|
|
248
|
+
rubyScript: v.optional(v.picklist(RUBY_SCRIPTS), DEFAULTS.plan.rubyScript),
|
|
249
|
+
/** The sources whose words get ruby. */
|
|
250
|
+
sources: v.optional(v.array(v.picklist(WORD_SOURCES)), () => [...WORD_SOURCES]),
|
|
251
|
+
/** The most common words left bare, by score: a reader knows them. Lone kanji count too. */
|
|
252
|
+
skip: v.optional(Count, DEFAULTS.plan.skip),
|
|
253
|
+
/** How many words get ruby after those skipped, by score; all of them when left out. */
|
|
254
|
+
take: v.optional(v.pipe(Count, v.minValue(1))),
|
|
255
|
+
/** Words on the JLPT lists of this level and every easier one stay bare. */
|
|
256
|
+
skipJlpt: v.optional(v.picklist(JLPT_LEVELS)),
|
|
257
|
+
/** Annotate a kanji alone in its run, or one no word claims after another kanji (a suffix); when off, only selectors give them ruby. */
|
|
258
|
+
isolated: v.optional(v.boolean(), DEFAULTS.plan.isolated),
|
|
259
|
+
/** A kanji alone in its run is annotated only when one reading has this share of the evidence. */
|
|
260
|
+
isolatedMinShare: v.optional(v.pipe(v.number(), v.minValue(.5), v.maxValue(1)), DEFAULTS.plan.isolatedMinShare),
|
|
261
|
+
words: v.optional(v.object({
|
|
262
|
+
/** Extra words in inline notation, `日[に]本[ほん]`. */
|
|
263
|
+
add: v.optional(v.array(v.string()), []),
|
|
264
|
+
/** Written forms never to annotate. */
|
|
265
|
+
remove: v.optional(v.array(v.string()), []),
|
|
266
|
+
/** Readings to make the default, in inline notation. */
|
|
267
|
+
pin: v.optional(v.array(v.string()), [])
|
|
268
|
+
}), {}),
|
|
269
|
+
metrics: v.optional(RubyMetricsSchema, {})
|
|
270
|
+
});
|
|
271
|
+
function resolveOptions(input = {}) {
|
|
272
|
+
return v.parse(PlanOptionsSchema, input);
|
|
273
|
+
}
|
|
274
|
+
const Codepoint = v.pipe(v.number(), v.integer(), v.minValue(0), v.maxValue(1114111));
|
|
275
|
+
/**
|
|
276
|
+
* The contract between the planner and the compiler. The compiler makes no
|
|
277
|
+
* decision about readings: it draws what `glyphs` describes and compiles `fea`.
|
|
278
|
+
* In `fea`, base characters are named `uniXXXX` / `uXXXXX` after their codepoint.
|
|
279
|
+
*/
|
|
280
|
+
const PlanSchema = v.object({
|
|
281
|
+
version: v.literal(1),
|
|
282
|
+
names: v.object({
|
|
283
|
+
family: v.string(),
|
|
284
|
+
description: v.string()
|
|
285
|
+
}),
|
|
286
|
+
metrics: v.object({
|
|
287
|
+
rubyScale: v.number(),
|
|
288
|
+
rubyGap: v.number(),
|
|
289
|
+
compressionFloor: v.number(),
|
|
290
|
+
maxRubyWidth: v.number()
|
|
291
|
+
}),
|
|
292
|
+
/** Base characters the rules name; each must be in the base font's cmap. */
|
|
293
|
+
codepoints: v.array(Codepoint),
|
|
294
|
+
/**
|
|
295
|
+
* Base + variation selector sequences to map in `cmap` format 14, each to an
|
|
296
|
+
* empty glyph named `name`, with the base's advance: the rules replace it.
|
|
297
|
+
*/
|
|
298
|
+
variants: v.array(v.object({
|
|
299
|
+
name: v.pipe(v.string(), v.regex(/^[A-Z_][\w.]{0,62}$/i)),
|
|
300
|
+
base: Codepoint,
|
|
301
|
+
selector: Codepoint
|
|
302
|
+
})),
|
|
303
|
+
glyphs: v.array(v.object({
|
|
304
|
+
name: v.pipe(v.string(), v.regex(/^[A-Z_][\w.]{0,62}$/i)),
|
|
305
|
+
/** The base characters drawn side by side; more than one makes a ligature. */
|
|
306
|
+
bases: v.pipe(v.array(Codepoint), v.minLength(1)),
|
|
307
|
+
/** Ruby text drawn above characters `[start, end)` of `bases`. */
|
|
308
|
+
ruby: v.array(v.object({
|
|
309
|
+
start: v.number(),
|
|
310
|
+
end: v.number(),
|
|
311
|
+
text: v.pipe(v.string(), v.nonEmpty())
|
|
312
|
+
}))
|
|
313
|
+
})),
|
|
314
|
+
fea: v.string(),
|
|
315
|
+
/** Subset the base font to these codepoints before building (live previews). */
|
|
316
|
+
subset: v.optional(v.array(Codepoint)),
|
|
317
|
+
/** Shaping checks the compiler runs on its output: `text` must shape to `glyphs`. */
|
|
318
|
+
smoke: v.array(v.object({
|
|
319
|
+
text: v.string(),
|
|
320
|
+
glyphs: v.array(v.string())
|
|
321
|
+
}))
|
|
322
|
+
});
|
|
323
|
+
/**
|
|
324
|
+
* The bounds of a reading's score, 0 to 100: above `tagged`, a word JMdict's
|
|
325
|
+
* lists tag, by its weight; above `used`, one Tatoeba's sentences use (20 and
|
|
326
|
+
* its uses, up to 50); above `listed`, one a manual list (15) or JMnedict (10)
|
|
327
|
+
* names; any other JMdict word scores `listed`.
|
|
328
|
+
*/
|
|
329
|
+
const SCORE = {
|
|
330
|
+
tagged: 50,
|
|
331
|
+
used: 20,
|
|
332
|
+
listed: 5
|
|
333
|
+
};
|
|
334
|
+
/** Score points per factor e of weight: a point is a twelfth, about 9%, of it. */
|
|
335
|
+
const POINTS = 12;
|
|
336
|
+
/**
|
|
337
|
+
* The score of a tagged reading of corpus weight `weight`, at most 1: 1 / its
|
|
338
|
+
* newspaper bucket, or a lone kanji's share of its uses. It is 100 for 1, 92
|
|
339
|
+
* for 1/2, 54 for 1/48; a weight too small to score above `tagged` scores 0.
|
|
340
|
+
*/
|
|
341
|
+
function scoreOf(weight) {
|
|
342
|
+
const score = Math.round(100 + POINTS * Math.log(weight));
|
|
343
|
+
return score > SCORE.tagged ? score : 0;
|
|
344
|
+
}
|
|
345
|
+
/** The corpus weight of a reading's score, which `scoreOf` gives; 0 for one no list tags. */
|
|
346
|
+
const weightOf = (score) => score > SCORE.tagged ? Math.exp((score - 100) / POINTS) : 0;
|
|
347
|
+
const isRecord = (value) => typeof value === "object" && value !== null && !Array.isArray(value);
|
|
348
|
+
const isReading = (value) => Array.isArray(value) && value.length === 2 && value.every(Number.isInteger) && value[0] >= 0 && value[0] <= 100 && value[1] > 0 && value[1] < 1 << SOURCES.length;
|
|
349
|
+
/** Readings by run or kanji, then by alignment or reading. Checked by hand: valibot takes seconds over the 400,000 readings of the dictionary. */
|
|
350
|
+
const ReadingsSchema = v.custom((input) => isRecord(input) && Object.values(input).every((readings) => isRecord(readings) && Object.values(readings).every(isReading)), "Expected readings by run, each [score, sources]");
|
|
351
|
+
/** The bit of each source, which the file writes for its readers. */
|
|
352
|
+
const SOURCE_BITS = Object.fromEntries(SOURCES.map((source) => [source, sourceBit(source)]));
|
|
353
|
+
/**
|
|
354
|
+
* The compiled dictionary, the contract between `@furifont/dictionary` and
|
|
355
|
+
* the planner: readings by the Han run a font sees, each with its score and
|
|
356
|
+
* sources. A run's readings come ranked, the default first. Run readings are
|
|
357
|
+
* JmdictFurigana alignments within the run.
|
|
358
|
+
*/
|
|
359
|
+
const DictionarySchema = v.object({
|
|
360
|
+
/** Source datasets and the version each was compiled from. */
|
|
361
|
+
versions: v.record(v.string(), v.string()),
|
|
362
|
+
/** The bit of each source in the readings' masks. */
|
|
363
|
+
sources: v.custom((input) => isRecord(input) && JSON.stringify(input) === JSON.stringify(SOURCE_BITS), "Expected the sources of this version of @furifont/core"),
|
|
364
|
+
/** Kanji to its on'yomi, in hiragana. */
|
|
365
|
+
onyomi: v.record(v.string(), v.array(v.string())),
|
|
366
|
+
/** Runs of two or more kanji, then their alignments: `{ 日本: { '0-1:にほん': [75, 61] } }`. */
|
|
367
|
+
runs: ReadingsSchema,
|
|
368
|
+
/** Kanji alone in a run, then their readings: `{ 来: { く: [98, 2101] } }`. */
|
|
369
|
+
kanji: ReadingsSchema,
|
|
370
|
+
/** Written forms never to annotate, such as 月, whose reading alone is a guess; users' removals add to them. */
|
|
371
|
+
bare: v.array(v.string())
|
|
372
|
+
});
|
|
373
|
+
/** The file `DictionarySchema` reads: an entry a line. */
|
|
374
|
+
function formatDictionary(dictionary) {
|
|
375
|
+
const lines = (table) => Object.entries(table).map(([key, value]) => `${JSON.stringify(key)}:${JSON.stringify(value)}`).join(",\n");
|
|
376
|
+
return `{
|
|
377
|
+
"versions":${JSON.stringify(dictionary.versions)},
|
|
378
|
+
"sources":${JSON.stringify(dictionary.sources)},
|
|
379
|
+
"bare":${JSON.stringify(dictionary.bare)},
|
|
380
|
+
"onyomi":{
|
|
381
|
+
${lines(dictionary.onyomi)}
|
|
382
|
+
},
|
|
383
|
+
"kanji":{
|
|
384
|
+
${lines(dictionary.kanji)}
|
|
385
|
+
},
|
|
386
|
+
"runs":{
|
|
387
|
+
${lines(dictionary.runs)}
|
|
388
|
+
}
|
|
389
|
+
}
|
|
390
|
+
`;
|
|
391
|
+
}
|
|
392
|
+
//#endregion
|
|
393
|
+
//#region src/fuse.ts
|
|
394
|
+
/** JMdict orders readings within a form; a run read inside a longer form ranks after the form's own readings. */
|
|
395
|
+
const OTHER_FORM_ORDER = 1e3;
|
|
396
|
+
const OKURIGANA = /[\u3041-\u3096]/;
|
|
397
|
+
/** Uses in Tatoeba's sentences that make a word one of the sentences'. */
|
|
398
|
+
const MIN_USES = 3;
|
|
399
|
+
/** With this many uses alone in the sentences, they decide how a lone kanji reads rather than the word lists. */
|
|
400
|
+
const LONE_MIN_USES = 20;
|
|
401
|
+
/** A tagged reading of a kanji that the sentences decide keeps this score at most, so it stays among the kanji's readings. */
|
|
402
|
+
const LONE_TAGGED_FLOOR = 60;
|
|
403
|
+
const PRIORITY_LISTS = [
|
|
404
|
+
"ichi",
|
|
405
|
+
"news",
|
|
406
|
+
"spec",
|
|
407
|
+
"gai"
|
|
408
|
+
];
|
|
409
|
+
const MANUAL_BITS = MANUAL_SOURCES.reduce((mask, name) => mask | sourceBit(name), 0);
|
|
410
|
+
/** The evidence of `text` read as `key`, met for the first time or not. */
|
|
411
|
+
function evidenceOf(table, text, key) {
|
|
412
|
+
return getOrCreate(getOrCreate(table, text, () => /* @__PURE__ */ new Map()), key, () => ({ sources: 0 }));
|
|
413
|
+
}
|
|
414
|
+
/** Spans as a JmdictFurigana alignment: `0:に;1:ほん`, `0-1:きょう`. */
|
|
415
|
+
function formatFurigana(spans) {
|
|
416
|
+
return spans.map((s) => `${s.end - s.start > 1 ? `${s.start}-${s.end - 1}` : s.start}:${s.ruby}`).join(";");
|
|
417
|
+
}
|
|
418
|
+
/** Rough corpus frequency of a tagged entry, in newspaper buckets of ~500 words each. */
|
|
419
|
+
function rankOf(entry) {
|
|
420
|
+
if (entry.priority) return entry.nf || Number(/nf(\d+)/.exec(entry.priority)?.[1] ?? 0) || (/[a-z]1\b/.test(entry.priority) ? 24 : 48);
|
|
421
|
+
}
|
|
422
|
+
/** Adds what a JMdict entry says of a reading: its lists, its rank and its JLPT level. */
|
|
423
|
+
function addEntry(into, entry) {
|
|
424
|
+
into.sources |= sourceBit("jmdict");
|
|
425
|
+
if (entry.jlpt) into.jlpt = Math.max(into.jlpt ?? 0, entry.jlpt);
|
|
426
|
+
const rank = rankOf(entry);
|
|
427
|
+
if (!rank) return;
|
|
428
|
+
into.rank = Math.min(into.rank ?? rank, rank);
|
|
429
|
+
if (entry.nf) into.sources |= sourceBit("nf");
|
|
430
|
+
for (const list of PRIORITY_LISTS.filter((list) => new RegExp(`\\b${list}[12]\\b`).test(entry.priority))) into.sources |= sourceBit(list);
|
|
431
|
+
}
|
|
432
|
+
function addRun(runs, run, entry) {
|
|
433
|
+
const evidence = evidenceOf(runs, run.text, formatFurigana(run.spans));
|
|
434
|
+
const order = run.text === entry.form ? entry.order : OTHER_FORM_ORDER + entry.order;
|
|
435
|
+
evidence.order = Math.min(evidence.order ?? order, order);
|
|
436
|
+
if (entry.tatoeba) evidence.uses = Math.max(evidence.uses ?? 0, entry.tatoeba);
|
|
437
|
+
addEntry(evidence, entry);
|
|
438
|
+
}
|
|
439
|
+
function addKanji(kanji, run, entry) {
|
|
440
|
+
const ruby = run.spans[0].ruby;
|
|
441
|
+
if (entry.tatoeba && !entry.conjugatesReading) {
|
|
442
|
+
const uses = getOrCreate(kanji.uses, run.text, () => /* @__PURE__ */ new Map());
|
|
443
|
+
uses.set(ruby, (uses.get(ruby) ?? 0) + entry.tatoeba);
|
|
444
|
+
}
|
|
445
|
+
if (rankOf(entry)) for (const r of entry.conjugatesReading && ruby === "く" ? [
|
|
446
|
+
"く",
|
|
447
|
+
"き",
|
|
448
|
+
"こ"
|
|
449
|
+
] : [ruby]) addEntry(evidenceOf(kanji.tagged, run.text, r), entry);
|
|
450
|
+
else if (OKURIGANA.test(entry.form[entry.form.indexOf(run.text) + run.text.length] ?? "")) getOrCreate(kanji.inflecting, run.text, () => /* @__PURE__ */ new Set()).add(ruby);
|
|
451
|
+
}
|
|
452
|
+
/** Its sources with the JLPT level and Tatoeba's sentences as bits. */
|
|
453
|
+
function sourcesOf(evidence) {
|
|
454
|
+
const level = evidence.jlpt && JLPT_LEVELS[5 - evidence.jlpt];
|
|
455
|
+
return evidence.sources | (level ? sourceBit(level) : 0) | ((evidence.uses ?? 0) >= MIN_USES ? sourceBit("tatoeba") : 0);
|
|
456
|
+
}
|
|
457
|
+
/** How common a run's reading is, by the most telling of its sources. */
|
|
458
|
+
function runScore(evidence) {
|
|
459
|
+
const uses = evidence.uses ?? 0;
|
|
460
|
+
return Math.max(evidence.rank ? scoreOf(1 / evidence.rank) : 0, uses >= MIN_USES ? SCORE.used + Math.min(uses, SCORE.tagged - SCORE.used) : 0, evidence.sources & MANUAL_BITS ? 15 : 0, evidence.sources & sourceBit("place") ? 10 : 0, evidence.order === void 0 ? 0 : SCORE.listed);
|
|
461
|
+
}
|
|
462
|
+
const tier = (score) => [
|
|
463
|
+
SCORE.tagged,
|
|
464
|
+
SCORE.used,
|
|
465
|
+
SCORE.listed
|
|
466
|
+
].filter((bound) => score > bound).length;
|
|
467
|
+
/**
|
|
468
|
+
* A run's readings ranked, the default first: tagged ones, then those the
|
|
469
|
+
* sentences use, each by JMdict's order; then the lists' and the place
|
|
470
|
+
* names', then the other JMdict words', each by score. Readings of the same
|
|
471
|
+
* kana are one, under the alignment that splits them most: 倉本 くら・もと
|
|
472
|
+
* rather than くらもと.
|
|
473
|
+
*/
|
|
474
|
+
function rankReadings(readings) {
|
|
475
|
+
const byKana = /* @__PURE__ */ new Map();
|
|
476
|
+
for (const [key, evidence] of readings) {
|
|
477
|
+
const spans = parseFurigana(key);
|
|
478
|
+
const reading = {
|
|
479
|
+
key,
|
|
480
|
+
spans,
|
|
481
|
+
score: runScore(evidence),
|
|
482
|
+
sources: sourcesOf(evidence),
|
|
483
|
+
order: evidence.order ?? Infinity
|
|
484
|
+
};
|
|
485
|
+
const kana = spans.map((s) => s.ruby).join("");
|
|
486
|
+
const same = byKana.get(kana);
|
|
487
|
+
if (!same) {
|
|
488
|
+
byKana.set(kana, reading);
|
|
489
|
+
continue;
|
|
490
|
+
}
|
|
491
|
+
const finer = reading.spans.length > same.spans.length ? reading : same;
|
|
492
|
+
byKana.set(kana, {
|
|
493
|
+
...finer,
|
|
494
|
+
score: Math.max(same.score, reading.score),
|
|
495
|
+
sources: same.sources | reading.sources,
|
|
496
|
+
order: Math.min(same.order, reading.order)
|
|
497
|
+
});
|
|
498
|
+
}
|
|
499
|
+
return [...byKana.values()].sort((a, b) => tier(b.score) - tier(a.score) || (tier(a.score) > 1 ? a.order - b.order || b.score - a.score : b.score - a.score || a.order - b.order)).map((r) => [r.key, [r.score, r.sources]]);
|
|
500
|
+
}
|
|
501
|
+
/**
|
|
502
|
+
* When the sentences use a kanji alone often enough to tell, each reading
|
|
503
|
+
* scores by its share of those uses, so that its weight in the planner is
|
|
504
|
+
* about that share; a tagged one keeps a little, to stay among its readings.
|
|
505
|
+
*/
|
|
506
|
+
function scoreUses(readings, uses) {
|
|
507
|
+
const total = [...uses.values()].reduce((a, b) => a + b, 0);
|
|
508
|
+
if (total < LONE_MIN_USES) return;
|
|
509
|
+
for (const reading of readings.values()) reading[0] = Math.min(reading[0], LONE_TAGGED_FLOOR);
|
|
510
|
+
for (const [ruby, n] of uses) {
|
|
511
|
+
const reading = getOrCreate(readings, ruby, () => [0, sourceBit("jmdict")]);
|
|
512
|
+
reading[0] = Math.max(reading[0], scoreOf(n / total));
|
|
513
|
+
if (n >= MIN_USES) reading[1] |= sourceBit("tatoeba");
|
|
514
|
+
}
|
|
515
|
+
}
|
|
516
|
+
/** A lone kanji's readings, the most common first: tagged words and the sentences score them; rare verbs add readings that score nothing. */
|
|
517
|
+
function kanjiReadings(kanji) {
|
|
518
|
+
const out = /* @__PURE__ */ new Map();
|
|
519
|
+
for (const [text, tagged] of kanji.tagged) {
|
|
520
|
+
const readings = new Map([...tagged].map(([ruby, evidence]) => [ruby, [scoreOf(1 / evidence.rank), sourcesOf(evidence)]]));
|
|
521
|
+
scoreUses(readings, kanji.uses.get(text) ?? /* @__PURE__ */ new Map());
|
|
522
|
+
out.set(text, new Map([...readings].filter(([, [score]]) => score).sort((a, b) => b[1][0] - a[1][0])));
|
|
523
|
+
}
|
|
524
|
+
for (const [text, rubies] of kanji.inflecting) {
|
|
525
|
+
const readings = getOrCreate(out, text, () => /* @__PURE__ */ new Map());
|
|
526
|
+
for (const ruby of rubies) {
|
|
527
|
+
const [score, sources] = readings.get(ruby) ?? [0, sourceBit("jmdict")];
|
|
528
|
+
readings.set(ruby, [score, sources | sourceBit("inflecting")]);
|
|
529
|
+
}
|
|
530
|
+
}
|
|
531
|
+
return out;
|
|
532
|
+
}
|
|
533
|
+
function notationRuns$1(notation) {
|
|
534
|
+
const { form, spans } = parseNotation(notation);
|
|
535
|
+
return hanRuns(form, spans);
|
|
536
|
+
}
|
|
537
|
+
/** The runs each list names, as the planner reads them: no lone kanji, which would be no evidence for the kanji's reading. */
|
|
538
|
+
function listedRuns(sources) {
|
|
539
|
+
const manual = MANUAL_SOURCES.flatMap((name) => (sources.manual[name] ?? []).flatMap(notationRuns$1).map((run) => [run, name]));
|
|
540
|
+
const places = sources.places.flatMap((place) => {
|
|
541
|
+
const runs = hanRuns(place.form, parseFurigana(place.furigana));
|
|
542
|
+
return runs.length === 1 && runs[0].text === place.form ? [[runs[0], "place"]] : [];
|
|
543
|
+
});
|
|
544
|
+
return [...manual, ...places].filter(([run]) => [...run.text].length > 1);
|
|
545
|
+
}
|
|
546
|
+
function addSource(readings, key, source) {
|
|
547
|
+
const [score, sources] = readings.get(key) ?? [0, 0];
|
|
548
|
+
readings.set(key, [score, sources | sourceBit(source)]);
|
|
549
|
+
}
|
|
550
|
+
/**
|
|
551
|
+
* Merges every source into the readings of the Han runs a font sees, each
|
|
552
|
+
* reading with its score and sources. What no plan can use is left out: a
|
|
553
|
+
* word's runs with an unannotated kanji, lone kanji of the lists or of words
|
|
554
|
+
* no list tags but before okurigana, and the on'yomi of kanji no reading uses.
|
|
555
|
+
*/
|
|
556
|
+
function fuseDictionary(sources) {
|
|
557
|
+
const runs = /* @__PURE__ */ new Map();
|
|
558
|
+
const kanji = {
|
|
559
|
+
tagged: /* @__PURE__ */ new Map(),
|
|
560
|
+
uses: /* @__PURE__ */ new Map(),
|
|
561
|
+
inflecting: /* @__PURE__ */ new Map()
|
|
562
|
+
};
|
|
563
|
+
for (const entry of sources.entries) for (const run of hanRuns(entry.form, parseFurigana(entry.furigana))) if ([...run.text].length > 1) addRun(runs, run, entry);
|
|
564
|
+
else addKanji(kanji, run, entry);
|
|
565
|
+
for (const [run, name] of listedRuns(sources)) evidenceOf(runs, run.text, formatFurigana(run.spans)).sources |= sourceBit(name);
|
|
566
|
+
const ranked = new Map([...runs].map(([text, readings]) => [text, new Map(rankReadings(readings))]));
|
|
567
|
+
const lone = kanjiReadings(kanji);
|
|
568
|
+
for (const run of sources.defaults.flatMap(notationRuns$1)) {
|
|
569
|
+
const [table, key] = [...run.text].length > 1 ? [ranked, formatFurigana(run.spans)] : [lone, run.spans[0].ruby];
|
|
570
|
+
addSource(getOrCreate(table, run.text, () => /* @__PURE__ */ new Map()), key, "default");
|
|
571
|
+
}
|
|
572
|
+
const used = new Set([...ranked.keys(), ...lone.keys()].flatMap((text) => [...text]));
|
|
573
|
+
const toRecord = (table) => Object.fromEntries([...table].map(([text, readings]) => [text, Object.fromEntries(readings)]));
|
|
574
|
+
return {
|
|
575
|
+
versions: sources.versions,
|
|
576
|
+
sources: SOURCE_BITS,
|
|
577
|
+
onyomi: Object.fromEntries(Object.entries(sources.onyomi).filter(([k]) => used.has(k))),
|
|
578
|
+
runs: toRecord(ranked),
|
|
579
|
+
kanji: toRecord(lone),
|
|
580
|
+
bare: sources.bare
|
|
581
|
+
};
|
|
582
|
+
}
|
|
182
583
|
//#endregion
|
|
183
584
|
//#region src/glyphs.ts
|
|
184
585
|
function glyphName(codepoint) {
|
|
@@ -529,94 +930,67 @@ const MIN_ALTERNATE_SHARE = .02;
|
|
|
529
930
|
/** A suffix reading is the one a kanji takes in this share of the runs it ends, out of at least `SUFFIX_MIN_RUNS`. */
|
|
530
931
|
const SUFFIX_MIN_SHARE = .9;
|
|
531
932
|
const SUFFIX_MIN_RUNS = 10;
|
|
532
|
-
/** With this many uses alone in the example sentences, they decide a lone kanji rather than the word lists. */
|
|
533
|
-
const LONE_MIN_USES = 20;
|
|
534
|
-
/** JMdict orders readings within a form; a run read inside a longer form ranks after the form's own readings. */
|
|
535
|
-
const OTHER_FORM_ORDER = 1e3;
|
|
536
933
|
const length = (text) => [...text].length;
|
|
537
934
|
function spanKey(spans) {
|
|
538
935
|
return spans.map((s) => `${s.start}-${s.end}:${s.ruby}`).join(";");
|
|
539
936
|
}
|
|
937
|
+
/** Pinned first; then readings that are evidence, before a place name's or a rare word's; then as the dictionary ranks them. */
|
|
540
938
|
function rank(a, b) {
|
|
541
|
-
return (b.pinned ?? 0) - (a.pinned ?? 0) || Number(
|
|
939
|
+
return (b.pinned ?? 0) - (a.pinned ?? 0) || Number(a.place ?? false) - Number(b.place ?? false) || a.order - b.order;
|
|
542
940
|
}
|
|
543
|
-
|
|
544
|
-
|
|
545
|
-
|
|
546
|
-
const
|
|
547
|
-
|
|
548
|
-
|
|
549
|
-
|
|
550
|
-
|
|
551
|
-
|
|
552
|
-
|
|
553
|
-
|
|
554
|
-
|
|
555
|
-
|
|
556
|
-
|
|
557
|
-
|
|
558
|
-
|
|
559
|
-
|
|
560
|
-
|
|
561
|
-
|
|
562
|
-
|
|
563
|
-
|
|
564
|
-
function indexRun(index, run, entry) {
|
|
565
|
-
const order = run.text === entry.form ? entry.order : OTHER_FORM_ORDER + entry.order;
|
|
566
|
-
const reading = {
|
|
567
|
-
spans: run.spans,
|
|
568
|
-
weight: entryWeight(entry),
|
|
569
|
-
tagged: !!entry.priority,
|
|
570
|
-
order,
|
|
571
|
-
nf: entry.priority ? entry.nf : 0,
|
|
572
|
-
jlpt: entry.jlpt ?? 0,
|
|
573
|
-
tags: priorityTags(entry),
|
|
574
|
-
tatoeba: entry.tatoeba ?? 0
|
|
575
|
-
};
|
|
576
|
-
const readings = getOrCreate(index.runs, run.text, () => /* @__PURE__ */ new Map());
|
|
577
|
-
const current = readings.get(spanKey(run.spans));
|
|
578
|
-
if (!current) {
|
|
579
|
-
readings.set(spanKey(run.spans), reading);
|
|
580
|
-
return;
|
|
941
|
+
const isJmdict = (reading) => !!(reading.sources & sourceBit("jmdict"));
|
|
942
|
+
const PIN = sourceBit("default");
|
|
943
|
+
function indexRuns(index, runs) {
|
|
944
|
+
for (const [text, readings] of Object.entries(runs)) {
|
|
945
|
+
const ranked = [];
|
|
946
|
+
for (const [furigana, [score, sources]] of Object.entries(readings)) {
|
|
947
|
+
const spans = parseFurigana(furigana);
|
|
948
|
+
if (sources & PIN) index.defaults.push({
|
|
949
|
+
text,
|
|
950
|
+
spans
|
|
951
|
+
});
|
|
952
|
+
if (sources & ~PIN) ranked.push({
|
|
953
|
+
spans,
|
|
954
|
+
weight: weightOf(score),
|
|
955
|
+
score,
|
|
956
|
+
sources,
|
|
957
|
+
order: ranked.length
|
|
958
|
+
});
|
|
959
|
+
}
|
|
960
|
+
index.runs.set(text, ranked);
|
|
961
|
+
if (ranked.some(isJmdict)) index.jmdict.set(text, ranked.filter(isJmdict));
|
|
581
962
|
}
|
|
582
|
-
keepBest(current, reading);
|
|
583
|
-
current.tagged ||= reading.tagged;
|
|
584
|
-
current.order = Math.min(current.order, reading.order);
|
|
585
|
-
current.tatoeba = Math.max(current.tatoeba ?? 0, reading.tatoeba);
|
|
586
963
|
}
|
|
587
|
-
function
|
|
588
|
-
const
|
|
589
|
-
|
|
590
|
-
|
|
591
|
-
|
|
592
|
-
|
|
593
|
-
|
|
594
|
-
|
|
595
|
-
|
|
596
|
-
|
|
597
|
-
|
|
598
|
-
|
|
599
|
-
|
|
600
|
-
|
|
601
|
-
|
|
602
|
-
|
|
603
|
-
else readings.set(r, value);
|
|
964
|
+
function indexKanji(index, kanji) {
|
|
965
|
+
for (const [text, readings] of Object.entries(kanji)) for (const [ruby, [score, sources]] of Object.entries(readings)) {
|
|
966
|
+
if (weightOf(score)) getOrCreate(index.singles, text, () => /* @__PURE__ */ new Map()).set(ruby, {
|
|
967
|
+
weight: weightOf(score),
|
|
968
|
+
score,
|
|
969
|
+
sources
|
|
970
|
+
});
|
|
971
|
+
if (sources & sourceBit("inflecting")) getOrCreate(index.inflecting, text, () => /* @__PURE__ */ new Set()).add(ruby);
|
|
972
|
+
if (sources & PIN) index.defaults.push({
|
|
973
|
+
text,
|
|
974
|
+
spans: [{
|
|
975
|
+
start: 0,
|
|
976
|
+
end: 1,
|
|
977
|
+
ruby
|
|
978
|
+
}]
|
|
979
|
+
});
|
|
604
980
|
}
|
|
605
981
|
}
|
|
606
|
-
|
|
607
|
-
function
|
|
608
|
-
|
|
609
|
-
|
|
610
|
-
|
|
611
|
-
|
|
612
|
-
|
|
613
|
-
|
|
614
|
-
|
|
615
|
-
|
|
616
|
-
|
|
617
|
-
|
|
618
|
-
else if (OKURIGANA.test(entry.form[entry.form.indexOf(run.text) + run.text.length] ?? "")) getOrCreate(index.inflecting, run.text, () => /* @__PURE__ */ new Set()).add(run.spans[0].ruby);
|
|
619
|
-
}
|
|
982
|
+
/** The compiled dictionary as the planner looks it up. */
|
|
983
|
+
function indexDictionary(dictionary) {
|
|
984
|
+
const index = {
|
|
985
|
+
runs: /* @__PURE__ */ new Map(),
|
|
986
|
+
jmdict: /* @__PURE__ */ new Map(),
|
|
987
|
+
singles: /* @__PURE__ */ new Map(),
|
|
988
|
+
inflecting: /* @__PURE__ */ new Map(),
|
|
989
|
+
defaults: []
|
|
990
|
+
};
|
|
991
|
+
indexRuns(index, dictionary.runs);
|
|
992
|
+
indexKanji(index, dictionary.kanji);
|
|
993
|
+
return index;
|
|
620
994
|
}
|
|
621
995
|
function createContext(dictionary, options, coverage) {
|
|
622
996
|
const covered = (text) => codepoints(text).every((cp) => coverage.has(cp));
|
|
@@ -627,20 +1001,22 @@ function createContext(dictionary, options, coverage) {
|
|
|
627
1001
|
if (options.rubyScript === "katakanaOn" && kanji.length === 1 && isOnyomi(span.ruby, dictionary.onyomi[kanji[0]] ?? [])) return toKatakana(span.ruby);
|
|
628
1002
|
return span.ruby;
|
|
629
1003
|
};
|
|
630
|
-
const known = options.skipJlpt ?
|
|
1004
|
+
const known = options.skipJlpt ? JLPT_LEVELS.slice(0, JLPT_LEVELS.indexOf(options.skipJlpt) + 1).reduce((mask, level) => mask | sourceBit(level), 0) : 0;
|
|
631
1005
|
return {
|
|
632
1006
|
options,
|
|
633
|
-
removed: /* @__PURE__ */ new Set([...dictionary.bare
|
|
1007
|
+
removed: /* @__PURE__ */ new Set([...dictionary.bare, ...options.words.remove]),
|
|
634
1008
|
covered,
|
|
635
|
-
|
|
636
|
-
skipped: (r) => r.nf > 0 && r.nf < options.nfMin || r.jlpt >= known,
|
|
1009
|
+
known: (sources) => !!(sources & known),
|
|
637
1010
|
render,
|
|
638
1011
|
renderable: (run) => covered(run.text) && run.spans.every((_, i) => covered(render(run, i)))
|
|
639
1012
|
};
|
|
640
1013
|
}
|
|
641
|
-
function
|
|
1014
|
+
function notationRuns(notation) {
|
|
642
1015
|
const { form, spans } = parseNotation(notation);
|
|
643
|
-
|
|
1016
|
+
return hanRuns(form, spans);
|
|
1017
|
+
}
|
|
1018
|
+
function addCustomRuns(patterns, lonePins, runs, pinned, ctx) {
|
|
1019
|
+
for (const run of runs.filter(ctx.renderable)) {
|
|
644
1020
|
if (length(run.text) === 1) {
|
|
645
1021
|
if (pinned) lonePins.set(run.text, run.spans[0].ruby);
|
|
646
1022
|
continue;
|
|
@@ -652,101 +1028,69 @@ function addCustomWord(patterns, lonePins, notation, pinned, ctx) {
|
|
|
652
1028
|
else readings.push({
|
|
653
1029
|
spans: run.spans,
|
|
654
1030
|
weight: 1,
|
|
655
|
-
|
|
1031
|
+
score: 100,
|
|
1032
|
+
sources: 0,
|
|
656
1033
|
order: Number.MAX_SAFE_INTEGER,
|
|
657
|
-
nf: 1,
|
|
658
|
-
jlpt: 0,
|
|
659
|
-
tags: /* @__PURE__ */ new Set(),
|
|
660
1034
|
pinned
|
|
661
1035
|
});
|
|
662
1036
|
}
|
|
663
1037
|
}
|
|
664
|
-
/**
|
|
665
|
-
function
|
|
666
|
-
|
|
667
|
-
|
|
668
|
-
|
|
669
|
-
|
|
670
|
-
|
|
671
|
-
|
|
672
|
-
|
|
673
|
-
|
|
674
|
-
|
|
675
|
-
tagged: false,
|
|
676
|
-
order: 0,
|
|
677
|
-
nf: 0,
|
|
678
|
-
jlpt: 0,
|
|
679
|
-
tags: /* @__PURE__ */ new Set(),
|
|
680
|
-
place: true
|
|
681
|
-
}]);
|
|
682
|
-
}
|
|
683
|
-
}
|
|
684
|
-
/** Words of the manual sources fill the runs no common word claims, nor a skipped one; they come before place names. */
|
|
685
|
-
function addManual(patterns, manual, skipped, ctx) {
|
|
686
|
-
const names = ctx.options.manual ?? Object.keys(manual);
|
|
687
|
-
for (const notation of names.flatMap((name) => manual[name] ?? [])) {
|
|
688
|
-
const { form, spans } = parseNotation(notation);
|
|
689
|
-
for (const run of hanRuns(form, spans).filter(ctx.renderable)) {
|
|
690
|
-
if (length(run.text) === 1 || patterns.has(run.text) || skipped.has(run.text) || ctx.removed.has(run.text)) continue;
|
|
691
|
-
patterns.set(run.text, [{
|
|
692
|
-
spans: run.spans,
|
|
693
|
-
weight: 0,
|
|
694
|
-
tagged: false,
|
|
695
|
-
order: 0,
|
|
696
|
-
nf: 0,
|
|
697
|
-
jlpt: 0,
|
|
698
|
-
tags: /* @__PURE__ */ new Set(),
|
|
699
|
-
place: true
|
|
700
|
-
}]);
|
|
701
|
-
}
|
|
702
|
-
}
|
|
703
|
-
}
|
|
704
|
-
/** Words no list ranks fill the runs nothing else claims, each with its first reading: they are no evidence for the kanji's own. */
|
|
705
|
-
function addUntagged(patterns, index, skipped, ctx) {
|
|
706
|
-
for (const [text, readings] of index.runs) {
|
|
707
|
-
if (patterns.has(text) || skipped.has(text) || ctx.removed.has(text)) continue;
|
|
708
|
-
const first = [...readings.values()].filter((r) => ctx.renderable({
|
|
709
|
-
text,
|
|
710
|
-
spans: r.spans
|
|
711
|
-
})).sort((a, b) => a.order - b.order)[0];
|
|
712
|
-
if (first) patterns.set(text, [{
|
|
713
|
-
...first,
|
|
714
|
-
place: true
|
|
715
|
-
}]);
|
|
716
|
-
}
|
|
1038
|
+
/** A run's readings from the plan's sources that the font can draw: those the lists or the sentences back, else the first, which is no evidence for its kanji's. */
|
|
1039
|
+
function wordReadings(text, readings, wanted, ctx) {
|
|
1040
|
+
const selected = readings.filter((r) => r.sources & wanted && ctx.renderable({
|
|
1041
|
+
text,
|
|
1042
|
+
spans: r.spans
|
|
1043
|
+
}));
|
|
1044
|
+
const backed = selected.filter((r) => r.score > SCORE.used);
|
|
1045
|
+
return backed.length ? backed.map((r) => ({ ...r })) : selected.slice(0, 1).map((r) => ({
|
|
1046
|
+
...r,
|
|
1047
|
+
place: true
|
|
1048
|
+
}));
|
|
717
1049
|
}
|
|
718
1050
|
/**
|
|
719
|
-
*
|
|
720
|
-
*
|
|
721
|
-
*
|
|
1051
|
+
* The words of the plan's sources, but those the reader knows, those the
|
|
1052
|
+
* user removes and, by score, the `skip` most common, words and lone kanji
|
|
1053
|
+
* together; then the `take` next words, and the user's own. A word is skipped
|
|
1054
|
+
* whole: were only its known reading dropped, 今日 would show こんにち.
|
|
722
1055
|
*/
|
|
723
|
-
function selectWords(index,
|
|
724
|
-
const
|
|
1056
|
+
function selectWords(index, ctx) {
|
|
1057
|
+
const wanted = ctx.options.sources.reduce((mask, source) => mask | sourceBit(source), 0);
|
|
725
1058
|
const skipped = /* @__PURE__ */ new Set();
|
|
1059
|
+
const skippedKanji = /* @__PURE__ */ new Set();
|
|
1060
|
+
const words = [];
|
|
726
1061
|
for (const [text, readings] of index.runs) {
|
|
727
1062
|
if (ctx.removed.has(text)) continue;
|
|
728
|
-
if (
|
|
1063
|
+
if (readings.some((r) => ctx.known(r.sources))) {
|
|
729
1064
|
skipped.add(text);
|
|
730
1065
|
continue;
|
|
731
1066
|
}
|
|
732
|
-
const
|
|
733
|
-
|
|
734
|
-
spans: r.spans
|
|
735
|
-
}));
|
|
736
|
-
if (common.length) patterns.set(text, common.map((r) => ({ ...r })));
|
|
1067
|
+
const chosen = wordReadings(text, readings, wanted, ctx);
|
|
1068
|
+
if (chosen.length) words.push([text, chosen]);
|
|
737
1069
|
}
|
|
738
|
-
|
|
739
|
-
|
|
740
|
-
|
|
1070
|
+
const kanji = [...index.singles].map(([text, readings]) => [text, [...readings.values()].reduce((a, b) => b.score > a.score ? b : a)]);
|
|
1071
|
+
for (const [text] of kanji.filter(([, top]) => ctx.known(top.sources))) skippedKanji.add(text);
|
|
1072
|
+
const byScore = [...words.map(([text, readings]) => ({
|
|
1073
|
+
text,
|
|
1074
|
+
score: Math.max(...readings.map((r) => r.score)),
|
|
1075
|
+
word: true
|
|
1076
|
+
})), ...kanji.filter(([text]) => !skippedKanji.has(text)).map(([text, top]) => ({
|
|
1077
|
+
text,
|
|
1078
|
+
score: top.score,
|
|
1079
|
+
word: false
|
|
1080
|
+
}))].sort((a, b) => b.score - a.score);
|
|
1081
|
+
for (const { text, word } of byScore.slice(0, ctx.options.skip)) (word ? skipped : skippedKanji).add(text);
|
|
1082
|
+
const taken = new Set(byScore.slice(ctx.options.skip).filter((item) => item.word).slice(0, ctx.options.take).map((item) => item.text));
|
|
1083
|
+
const patterns = new Map(words.filter(([text]) => taken.has(text)));
|
|
741
1084
|
const lonePins = /* @__PURE__ */ new Map();
|
|
742
|
-
|
|
743
|
-
|
|
744
|
-
|
|
1085
|
+
addCustomRuns(patterns, lonePins, ctx.options.words.add.flatMap(notationRuns), 0, ctx);
|
|
1086
|
+
addCustomRuns(patterns, lonePins, index.defaults, 1, ctx);
|
|
1087
|
+
addCustomRuns(patterns, lonePins, ctx.options.words.pin.flatMap(notationRuns), 2, ctx);
|
|
745
1088
|
for (const readings of patterns.values()) readings.sort(rank).splice(15);
|
|
746
1089
|
return {
|
|
747
1090
|
patterns,
|
|
748
1091
|
lonePins,
|
|
749
|
-
skipped
|
|
1092
|
+
skipped,
|
|
1093
|
+
skippedKanji
|
|
750
1094
|
};
|
|
751
1095
|
}
|
|
752
1096
|
/** Readings of a lone kanji the font can draw, heaviest first, with their share of the evidence. */
|
|
@@ -766,24 +1110,6 @@ function loneReadings(kanji, readings, ctx) {
|
|
|
766
1110
|
share: r.weight / total
|
|
767
1111
|
})).sort((a, b) => b.weight - a.weight);
|
|
768
1112
|
}
|
|
769
|
-
/** The reading running text gives a lone kanji most, with its share of the uses, when the sentences use the kanji alone often enough to tell. */
|
|
770
|
-
function usedReading(kanji, uses, ctx) {
|
|
771
|
-
const drawable = [...uses ?? []].filter(([ruby]) => ctx.renderable({
|
|
772
|
-
text: kanji,
|
|
773
|
-
spans: [{
|
|
774
|
-
start: 0,
|
|
775
|
-
end: 1,
|
|
776
|
-
ruby
|
|
777
|
-
}]
|
|
778
|
-
}));
|
|
779
|
-
const total = drawable.reduce((sum, [, n]) => sum + n, 0);
|
|
780
|
-
if (ctx.options.minTatoeba === 0 || total < LONE_MIN_USES) return;
|
|
781
|
-
const [ruby, n] = drawable.sort((a, b) => b[1] - a[1])[0];
|
|
782
|
-
return {
|
|
783
|
-
ruby,
|
|
784
|
-
share: n / total
|
|
785
|
-
};
|
|
786
|
-
}
|
|
787
1113
|
function addWordEvidence(patterns, add) {
|
|
788
1114
|
for (const [text, readings] of patterns) {
|
|
789
1115
|
const chars = [...text];
|
|
@@ -803,12 +1129,13 @@ function addRareVerbEvidence(inflecting, ctx, add) {
|
|
|
803
1129
|
}
|
|
804
1130
|
/**
|
|
805
1131
|
* A kanji alone in its run gets ruby only when one reading clearly dominates
|
|
806
|
-
* the common words it stands alone in,
|
|
1132
|
+
* the common words it stands alone in, or the sentences' uses of it alone
|
|
1133
|
+
* (ゴミ箱 is common, yet 箱 reads はこ), whatever the word list's size: verbs
|
|
807
1134
|
* like 書く rank low in newspapers yet decide how 書 reads. Every kanji also
|
|
808
1135
|
* gets an ordered list of its readings, which the variation selectors index
|
|
809
1136
|
* into, except a kanji whose dominant reading the reader is taken to know.
|
|
810
1137
|
*/
|
|
811
|
-
function chooseLoneKanji(index, patterns, lonePins, ctx) {
|
|
1138
|
+
function chooseLoneKanji(index, { patterns, lonePins, skippedKanji }, ctx) {
|
|
812
1139
|
const isolated = new Map(ctx.options.isolated ? lonePins : []);
|
|
813
1140
|
const skipped = /* @__PURE__ */ new Set();
|
|
814
1141
|
const evidence = /* @__PURE__ */ new Map();
|
|
@@ -821,9 +1148,8 @@ function chooseLoneKanji(index, patterns, lonePins, ctx) {
|
|
|
821
1148
|
if (ctx.removed.has(kanji)) continue;
|
|
822
1149
|
const ranked = loneReadings(kanji, readings, ctx).filter((r) => r.share >= MIN_ALTERNATE_SHARE);
|
|
823
1150
|
const top = ranked[0];
|
|
824
|
-
|
|
825
|
-
if (top &&
|
|
826
|
-
else if (chosen && chosen.share >= ctx.options.isolatedMinShare && ctx.options.isolated && !isolated.has(kanji)) isolated.set(kanji, chosen.ruby);
|
|
1151
|
+
if (top && !lonePins.has(kanji) && skippedKanji.has(kanji)) skipped.add(kanji);
|
|
1152
|
+
else if (top && top.share >= ctx.options.isolatedMinShare && ctx.options.isolated && !isolated.has(kanji)) isolated.set(kanji, top.ruby);
|
|
827
1153
|
for (const r of ranked) add(kanji, r.ruby, r.weight);
|
|
828
1154
|
}
|
|
829
1155
|
addWordEvidence(patterns, add);
|
|
@@ -854,11 +1180,11 @@ function chooseSuffixes(index, ctx) {
|
|
|
854
1180
|
const count = (counts, key) => void counts.set(key, (counts.get(key) ?? 0) + 1);
|
|
855
1181
|
const starts = /* @__PURE__ */ new Map();
|
|
856
1182
|
const ends = /* @__PURE__ */ new Map();
|
|
857
|
-
for (const [text, readings] of index.
|
|
1183
|
+
for (const [text, readings] of index.jmdict) {
|
|
858
1184
|
const chars = [...text];
|
|
859
1185
|
count(starts, chars[0]);
|
|
860
1186
|
count(ends, chars.at(-1));
|
|
861
|
-
for (const { spans } of readings
|
|
1187
|
+
for (const { spans } of readings) {
|
|
862
1188
|
const span = spans.at(-1);
|
|
863
1189
|
if (span.end - span.start === 1) count(getOrCreate(evidence, chars.at(-1), () => /* @__PURE__ */ new Map()), span.ruby);
|
|
864
1190
|
}
|
|
@@ -922,10 +1248,10 @@ function findMisreadRuns(index, patterns, kanjiReadings, suffixes, skipped, ctx)
|
|
|
922
1248
|
}
|
|
923
1249
|
const protect = (text, readings, longest) => {
|
|
924
1250
|
const chars = [...text];
|
|
925
|
-
const candidates = [...new Set(
|
|
926
|
-
const expected =
|
|
1251
|
+
const candidates = [...new Set(readings.map((r) => assignment(r.spans, chars.length).join("|")))];
|
|
1252
|
+
const expected = readings.map((r) => assignment(r.spans, chars.length));
|
|
927
1253
|
if (!expected.some((e) => misreads(chars, e, known, longest, suffixes))) return false;
|
|
928
|
-
const truth =
|
|
1254
|
+
const truth = readings.toSorted(rank)[0];
|
|
929
1255
|
const run = {
|
|
930
1256
|
text,
|
|
931
1257
|
spans: truth.spans
|
|
@@ -939,7 +1265,7 @@ function findMisreadRuns(index, patterns, kanjiReadings, suffixes, skipped, ctx)
|
|
|
939
1265
|
let longest = 2;
|
|
940
1266
|
for (const text of known.keys()) longest = Math.max(longest, length(text));
|
|
941
1267
|
changed = false;
|
|
942
|
-
for (const [text, readings] of index.
|
|
1268
|
+
for (const [text, readings] of index.jmdict) if (!known.has(text) && ctx.covered(text) && protect(text, readings, longest)) changed = true;
|
|
943
1269
|
}
|
|
944
1270
|
return protectedRuns;
|
|
945
1271
|
}
|
|
@@ -1064,7 +1390,7 @@ function buildRules(patterns, kanjiReadings, protectedRuns, glyphs) {
|
|
|
1064
1390
|
};
|
|
1065
1391
|
}
|
|
1066
1392
|
function smokeChecks(patterns, lone, glyphs) {
|
|
1067
|
-
const byFrequency = [...patterns].sort((a, b) =>
|
|
1393
|
+
const byFrequency = [...patterns].sort((a, b) => b[1][0].score - a[1][0].score);
|
|
1068
1394
|
const checks = [];
|
|
1069
1395
|
for (const grouped of [false, true]) for (const [text, readings] of byFrequency.filter(([text, r]) => isGrouped({
|
|
1070
1396
|
text,
|
|
@@ -1094,24 +1420,19 @@ function smokeChecks(patterns, lone, glyphs) {
|
|
|
1094
1420
|
return checks;
|
|
1095
1421
|
}
|
|
1096
1422
|
/**
|
|
1097
|
-
* Indexes the dictionary once
|
|
1423
|
+
* Indexes the compiled dictionary once, then plans fonts from it. Every
|
|
1098
1424
|
* decision about readings happens here: which words get ruby, which reading
|
|
1099
1425
|
* is the default, which longer words must be protected from a shorter match.
|
|
1100
1426
|
*/
|
|
1101
1427
|
function createFontPlanner(dictionary) {
|
|
1102
|
-
const index =
|
|
1103
|
-
|
|
1104
|
-
singles: /* @__PURE__ */ new Map(),
|
|
1105
|
-
inflecting: /* @__PURE__ */ new Map(),
|
|
1106
|
-
loneUses: /* @__PURE__ */ new Map()
|
|
1107
|
-
};
|
|
1108
|
-
for (const entry of dictionary.entries) indexEntry(index, entry);
|
|
1109
|
-
const sources = Object.entries(dictionary.sources).map(([name, version]) => `${name} ${version}`).join(", ");
|
|
1428
|
+
const index = indexDictionary(dictionary);
|
|
1429
|
+
const sources = Object.entries(dictionary.versions).map(([name, version]) => `${name} ${version}`).join(", ");
|
|
1110
1430
|
/** Every decision for one set of options, over the whole dictionary: slow, and the same whatever the preview text. */
|
|
1111
1431
|
function analyze(options, coverage) {
|
|
1112
1432
|
const ctx = createContext(dictionary, options, coverage);
|
|
1113
|
-
const
|
|
1114
|
-
const {
|
|
1433
|
+
const selection = selectWords(index, ctx);
|
|
1434
|
+
const { patterns, skipped } = selection;
|
|
1435
|
+
const { isolated, kanjiReadings } = chooseLoneKanji(index, selection, ctx);
|
|
1115
1436
|
const suffixes = chooseSuffixes(index, ctx);
|
|
1116
1437
|
const protectedRuns = findMisreadRuns(index, patterns, kanjiReadings, suffixes, skipped, ctx);
|
|
1117
1438
|
const glyphs = createGlyphRegistry(ctx.render);
|
|
@@ -1246,148 +1567,4 @@ function createFontPlanner(dictionary) {
|
|
|
1246
1567
|
};
|
|
1247
1568
|
}
|
|
1248
1569
|
//#endregion
|
|
1249
|
-
|
|
1250
|
-
const RUBY_SCRIPTS = [
|
|
1251
|
-
"hiragana",
|
|
1252
|
-
"katakanaOn",
|
|
1253
|
-
"hepburn"
|
|
1254
|
-
];
|
|
1255
|
-
const PRIORITY_TAGS = [
|
|
1256
|
-
"ichi1",
|
|
1257
|
-
"ichi2",
|
|
1258
|
-
"news1",
|
|
1259
|
-
"news2",
|
|
1260
|
-
"spec1",
|
|
1261
|
-
"spec2",
|
|
1262
|
-
"gai1",
|
|
1263
|
-
"gai2"
|
|
1264
|
-
];
|
|
1265
|
-
/** Easiest first. */
|
|
1266
|
-
/** The dictionary's manually maintained sources, which a plan may include by name. */
|
|
1267
|
-
const MANUAL_SOURCES = [
|
|
1268
|
-
"mahjong-yaku",
|
|
1269
|
-
"jr-stations",
|
|
1270
|
-
"yojijukugo"
|
|
1271
|
-
];
|
|
1272
|
-
const JLPT_LEVELS = [
|
|
1273
|
-
"N5",
|
|
1274
|
-
"N4",
|
|
1275
|
-
"N3",
|
|
1276
|
-
"N2",
|
|
1277
|
-
"N1"
|
|
1278
|
-
];
|
|
1279
|
-
/** Sizes in em, relative to the base font. */
|
|
1280
|
-
const RubyMetricsSchema = v.object({
|
|
1281
|
-
/** Ruby size relative to the base characters. */
|
|
1282
|
-
rubyScale: v.optional(v.pipe(v.number(), v.minValue(.2), v.maxValue(.8)), DEFAULTS.plan.metrics.rubyScale),
|
|
1283
|
-
/** Space between the top of the em box and the bottom of the ruby. */
|
|
1284
|
-
rubyGap: v.optional(v.pipe(v.number(), v.minValue(0), v.maxValue(.5)), DEFAULTS.plan.metrics.rubyGap),
|
|
1285
|
-
/** Ruby is squeezed horizontally down to this scale before it may overhang. */
|
|
1286
|
-
compressionFloor: v.optional(v.pipe(v.number(), v.minValue(.3), v.maxValue(1)), DEFAULTS.plan.metrics.compressionFloor),
|
|
1287
|
-
/** Widest a ruby may get, overhanging its base symmetrically. */
|
|
1288
|
-
maxRubyWidth: v.optional(v.pipe(v.number(), v.minValue(1), v.maxValue(3)), DEFAULTS.plan.metrics.maxRubyWidth)
|
|
1289
|
-
});
|
|
1290
|
-
const Bucket = v.pipe(v.number(), v.integer(), v.minValue(1), v.maxValue(48));
|
|
1291
|
-
/** `schema`, checking that its frequency window `nfMin..nfMax` holds a bucket; for schemas that extend the options too. */
|
|
1292
|
-
function withNfWindow(schema) {
|
|
1293
|
-
return v.pipe(schema, v.check((o) => o.nfMin <= o.nfMax, "nfMin must not exceed nfMax"));
|
|
1294
|
-
}
|
|
1295
|
-
const PlanOptionsSchema = withNfWindow(v.object({
|
|
1296
|
-
family: v.optional(v.pipe(v.string(), v.trim(), v.nonEmpty(), v.check((name) => !/source/i.test(name), "Source is a reserved font name of the base font")), DEFAULTS.plan.family),
|
|
1297
|
-
rubyScript: v.optional(v.picklist(RUBY_SCRIPTS), DEFAULTS.plan.rubyScript),
|
|
1298
|
-
/**
|
|
1299
|
-
* Words whose newspaper frequency bucket is below this stay bare: the most
|
|
1300
|
-
* common ones, which a reader knows. Tags don't bring them back.
|
|
1301
|
-
*/
|
|
1302
|
-
nfMin: v.optional(Bucket, DEFAULTS.plan.nfMin),
|
|
1303
|
-
/** Words whose newspaper frequency bucket is at most this are annotated. */
|
|
1304
|
-
nfMax: v.optional(Bucket, DEFAULTS.plan.nfMax),
|
|
1305
|
-
/**
|
|
1306
|
-
* JMdict priority tags that select a word whatever its frequency bucket:
|
|
1307
|
-
* `spec1` catches words like 日本 that the newspaper ranking misses.
|
|
1308
|
-
*/
|
|
1309
|
-
selectTags: v.optional(v.array(v.picklist(PRIORITY_TAGS)), () => [...DEFAULTS.plan.selectTags]),
|
|
1310
|
-
/**
|
|
1311
|
-
* Words of two or more kanji that Tatoeba's example sentences use at least
|
|
1312
|
-
* this many times are annotated whatever their bucket: everyday words that
|
|
1313
|
-
* no list tags, such as 留守中, which would otherwise be bare or read as a
|
|
1314
|
-
* place name (国中 くになか, not くにじゅう). The sentences also decide how a
|
|
1315
|
-
* lone kanji reads, where they use it alone often enough. 0 turns them off.
|
|
1316
|
-
*/
|
|
1317
|
-
minTatoeba: v.optional(v.pipe(v.number(), v.integer(), v.minValue(0)), DEFAULTS.plan.minTatoeba),
|
|
1318
|
-
/** Words on the JLPT lists of this level and every easier one stay bare. */
|
|
1319
|
-
skipJlpt: v.optional(v.picklist(JLPT_LEVELS)),
|
|
1320
|
-
/** Annotate place names from the dictionary where no common word claims the run. */
|
|
1321
|
-
places: v.optional(v.boolean(), DEFAULTS.plan.places),
|
|
1322
|
-
/** Annotate the words of JMdict that no list ranks or tags (such as 四字熟語), where no common word, manual list or place name claims the run; each gets its first reading. */
|
|
1323
|
-
untagged: v.optional(v.boolean(), DEFAULTS.plan.untagged),
|
|
1324
|
-
/** The dictionary's manually maintained sources (such as `mahjong-yaku`) to annotate; all of them when left out. */
|
|
1325
|
-
manual: v.optional(v.array(v.picklist(MANUAL_SOURCES))),
|
|
1326
|
-
/** Annotate a kanji alone in its run, or one no word claims after another kanji (a suffix); when off, only selectors give them ruby. */
|
|
1327
|
-
isolated: v.optional(v.boolean(), DEFAULTS.plan.isolated),
|
|
1328
|
-
/** A kanji alone in its run is annotated only when one reading has this share of the evidence. */
|
|
1329
|
-
isolatedMinShare: v.optional(v.pipe(v.number(), v.minValue(.5), v.maxValue(1)), DEFAULTS.plan.isolatedMinShare),
|
|
1330
|
-
words: v.optional(v.object({
|
|
1331
|
-
/** Extra words in inline notation, `日[に]本[ほん]`. */
|
|
1332
|
-
add: v.optional(v.array(v.string()), []),
|
|
1333
|
-
/** Written forms never to annotate. */
|
|
1334
|
-
remove: v.optional(v.array(v.string()), []),
|
|
1335
|
-
/** Readings to make the default, in inline notation. */
|
|
1336
|
-
pin: v.optional(v.array(v.string()), [])
|
|
1337
|
-
}), {}),
|
|
1338
|
-
metrics: v.optional(RubyMetricsSchema, {})
|
|
1339
|
-
}));
|
|
1340
|
-
function resolveOptions(input = {}) {
|
|
1341
|
-
return v.parse(PlanOptionsSchema, input);
|
|
1342
|
-
}
|
|
1343
|
-
const Codepoint = v.pipe(v.number(), v.integer(), v.minValue(0), v.maxValue(1114111));
|
|
1344
|
-
/**
|
|
1345
|
-
* The contract between the planner and the compiler. The compiler makes no
|
|
1346
|
-
* decision about readings: it draws what `glyphs` describes and compiles `fea`.
|
|
1347
|
-
* In `fea`, base characters are named `uniXXXX` / `uXXXXX` after their codepoint.
|
|
1348
|
-
*/
|
|
1349
|
-
const PlanSchema = v.object({
|
|
1350
|
-
version: v.literal(1),
|
|
1351
|
-
names: v.object({
|
|
1352
|
-
family: v.string(),
|
|
1353
|
-
description: v.string()
|
|
1354
|
-
}),
|
|
1355
|
-
metrics: v.object({
|
|
1356
|
-
rubyScale: v.number(),
|
|
1357
|
-
rubyGap: v.number(),
|
|
1358
|
-
compressionFloor: v.number(),
|
|
1359
|
-
maxRubyWidth: v.number()
|
|
1360
|
-
}),
|
|
1361
|
-
/** Base characters the rules name; each must be in the base font's cmap. */
|
|
1362
|
-
codepoints: v.array(Codepoint),
|
|
1363
|
-
/**
|
|
1364
|
-
* Base + variation selector sequences to map in `cmap` format 14, each to an
|
|
1365
|
-
* empty glyph named `name`, with the base's advance: the rules replace it.
|
|
1366
|
-
*/
|
|
1367
|
-
variants: v.array(v.object({
|
|
1368
|
-
name: v.pipe(v.string(), v.regex(/^[A-Z_][\w.]{0,62}$/i)),
|
|
1369
|
-
base: Codepoint,
|
|
1370
|
-
selector: Codepoint
|
|
1371
|
-
})),
|
|
1372
|
-
glyphs: v.array(v.object({
|
|
1373
|
-
name: v.pipe(v.string(), v.regex(/^[A-Z_][\w.]{0,62}$/i)),
|
|
1374
|
-
/** The base characters drawn side by side; more than one makes a ligature. */
|
|
1375
|
-
bases: v.pipe(v.array(Codepoint), v.minLength(1)),
|
|
1376
|
-
/** Ruby text drawn above characters `[start, end)` of `bases`. */
|
|
1377
|
-
ruby: v.array(v.object({
|
|
1378
|
-
start: v.number(),
|
|
1379
|
-
end: v.number(),
|
|
1380
|
-
text: v.pipe(v.string(), v.nonEmpty())
|
|
1381
|
-
}))
|
|
1382
|
-
})),
|
|
1383
|
-
fea: v.string(),
|
|
1384
|
-
/** Subset the base font to these codepoints before building (live previews). */
|
|
1385
|
-
subset: v.optional(v.array(Codepoint)),
|
|
1386
|
-
/** Shaping checks the compiler runs on its output: `text` must shape to `glyphs`. */
|
|
1387
|
-
smoke: v.array(v.object({
|
|
1388
|
-
text: v.string(),
|
|
1389
|
-
glyphs: v.array(v.string())
|
|
1390
|
-
}))
|
|
1391
|
-
});
|
|
1392
|
-
//#endregion
|
|
1393
|
-
export { DEFAULTS, JLPT_LEVELS, MANUAL_SOURCES, MAX_READINGS, PREBUILT, PRIORITY_TAGS, PlanOptionsSchema, PlanSchema, RUBY_SCRIPTS, SELECTOR_FIRST_READING, SELECTOR_SUPPRESS, VERSION_PROBE, createFontPlanner, formatNotation, glyphName, hanRuns, isHan, isOnyomi, parseFurigana, parseNotation, resolveOptions, toHepburn, toHiragana, toKatakana, withNfWindow };
|
|
1570
|
+
export { DEFAULTS, DictionarySchema, JLPT_LEVELS, MANUAL_SOURCES, MAX_READINGS, PREBUILT, PlanOptionsSchema, PlanSchema, RUBY_SCRIPTS, SCORE, SELECTOR_FIRST_READING, SELECTOR_SUPPRESS, SOURCES, VERSION_PROBE, WORD_SOURCES, createFontPlanner, formatDictionary, formatFurigana, formatNotation, fuseDictionary, glyphName, hanRuns, isHan, isOnyomi, parseFurigana, parseNotation, resolveOptions, scoreOf, sourceBit, sourcesIn, toHepburn, toHiragana, toKatakana, weightOf };
|