@furifont/core 0.2.0 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.d.mts +156 -128
- package/dist/index.mjs +545 -321
- package/package.json +1 -1
package/dist/index.mjs
CHANGED
|
@@ -10,19 +10,8 @@ const DEFAULTS = {
|
|
|
10
10
|
plan: {
|
|
11
11
|
family: "Furifont Sans",
|
|
12
12
|
rubyScript: "hiragana",
|
|
13
|
-
/**
|
|
14
|
-
|
|
15
|
-
nfMax: 48,
|
|
16
|
-
/** JMdict priority tags that select a word whatever its bucket: the first tier of each list, which JMdict marks as common. */
|
|
17
|
-
selectTags: [
|
|
18
|
-
"ichi1",
|
|
19
|
-
"news1",
|
|
20
|
-
"spec1",
|
|
21
|
-
"gai1"
|
|
22
|
-
],
|
|
23
|
-
/** Uses in Tatoeba's example sentences that select a word whatever its bucket: about 1,000 more words, and 50 glyphs. */
|
|
24
|
-
minTatoeba: 3,
|
|
25
|
-
places: true,
|
|
13
|
+
/** No word is left bare for being common; every word of every source gets ruby. */
|
|
14
|
+
skip: 0,
|
|
26
15
|
isolated: true,
|
|
27
16
|
isolatedMinShare: .8,
|
|
28
17
|
/** Sizes in em, relative to the base font. */
|
|
@@ -81,7 +70,7 @@ const PREBUILT = {
|
|
|
81
70
|
style: "VF",
|
|
82
71
|
weights: [250, 900]
|
|
83
72
|
},
|
|
84
|
-
/** Options set on every prebuilt font over the defaults above, such as `{
|
|
73
|
+
/** Options set on every prebuilt font over the defaults above, such as `{ take: 20000 }`. */
|
|
85
74
|
options: {}
|
|
86
75
|
};
|
|
87
76
|
/**
|
|
@@ -100,16 +89,22 @@ const HAN = /\p{Script=Han}/u;
|
|
|
100
89
|
function isHan(char) {
|
|
101
90
|
return HAN.test(char);
|
|
102
91
|
}
|
|
92
|
+
/** Spans of a JmdictFurigana alignment; a loop, not splits, as the planner parses the 400,000 readings of the dictionary as it starts. */
|
|
103
93
|
function parseFurigana(furigana) {
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
94
|
+
const spans = [];
|
|
95
|
+
for (const part of furigana.split(";")) {
|
|
96
|
+
if (!part) continue;
|
|
97
|
+
const colon = part.indexOf(":");
|
|
98
|
+
const range = colon < 0 ? part : part.slice(0, colon);
|
|
99
|
+
const dash = range.indexOf("-");
|
|
100
|
+
const start = Number(dash < 0 ? range : range.slice(0, dash));
|
|
101
|
+
spans.push({
|
|
108
102
|
start,
|
|
109
|
-
end:
|
|
110
|
-
ruby
|
|
111
|
-
};
|
|
112
|
-
}
|
|
103
|
+
end: (dash < 0 ? start : Number(range.slice(dash + 1))) + 1,
|
|
104
|
+
ruby: colon < 0 ? "" : part.slice(colon + 1)
|
|
105
|
+
});
|
|
106
|
+
}
|
|
107
|
+
return spans;
|
|
113
108
|
}
|
|
114
109
|
/**
|
|
115
110
|
* Splits a form into its Han runs. Browsers itemize text by script before
|
|
@@ -178,6 +173,413 @@ function formatNotation(text, spans) {
|
|
|
178
173
|
}
|
|
179
174
|
return out + chars.slice(at).join("");
|
|
180
175
|
}
|
|
176
|
+
function getOrCreate(map, key, create) {
|
|
177
|
+
let value = map.get(key);
|
|
178
|
+
if (value === void 0) map.set(key, value = create());
|
|
179
|
+
return value;
|
|
180
|
+
}
|
|
181
|
+
//#endregion
|
|
182
|
+
//#region src/schema.ts
|
|
183
|
+
const RUBY_SCRIPTS = [
|
|
184
|
+
"hiragana",
|
|
185
|
+
"katakanaOn",
|
|
186
|
+
"hepburn"
|
|
187
|
+
];
|
|
188
|
+
/** The dictionary's manually maintained sources. */
|
|
189
|
+
const MANUAL_SOURCES = [
|
|
190
|
+
"mahjong-yaku",
|
|
191
|
+
"jr-stations",
|
|
192
|
+
"yojijukugo",
|
|
193
|
+
"japan-post"
|
|
194
|
+
];
|
|
195
|
+
/**
|
|
196
|
+
* The sources a plan takes words from: any JMdict word; JMdict's priority
|
|
197
|
+
* lists (both tiers of each) and its newspaper ranking; words Tatoeba's
|
|
198
|
+
* sentences use at least three times; JMnedict's place names; the manual lists.
|
|
199
|
+
*/
|
|
200
|
+
const WORD_SOURCES = [
|
|
201
|
+
"jmdict",
|
|
202
|
+
"ichi",
|
|
203
|
+
"news",
|
|
204
|
+
"spec",
|
|
205
|
+
"gai",
|
|
206
|
+
"nf",
|
|
207
|
+
"tatoeba",
|
|
208
|
+
"place",
|
|
209
|
+
...MANUAL_SOURCES
|
|
210
|
+
];
|
|
211
|
+
/** Easiest first. */
|
|
212
|
+
const JLPT_LEVELS = [
|
|
213
|
+
"N5",
|
|
214
|
+
"N4",
|
|
215
|
+
"N3",
|
|
216
|
+
"N2",
|
|
217
|
+
"N1"
|
|
218
|
+
];
|
|
219
|
+
/**
|
|
220
|
+
* Every source a reading may have, a bit each in this order: the word
|
|
221
|
+
* sources, then the easiest JLPT level listing it, `inflecting` (a kanji read
|
|
222
|
+
* before okurigana in a JMdict word no list tags, such as 奉る) and `default`,
|
|
223
|
+
* the dictionary's curated default.
|
|
224
|
+
*/
|
|
225
|
+
const SOURCES = [
|
|
226
|
+
...WORD_SOURCES,
|
|
227
|
+
...JLPT_LEVELS,
|
|
228
|
+
"inflecting",
|
|
229
|
+
"default"
|
|
230
|
+
];
|
|
231
|
+
const sourceBit = (source) => 1 << SOURCES.indexOf(source);
|
|
232
|
+
/** The sources a mask of `SOURCES` bits names. */
|
|
233
|
+
const sourcesIn = (mask) => SOURCES.filter((source) => mask & sourceBit(source));
|
|
234
|
+
/** Sizes in em, relative to the base font. */
|
|
235
|
+
const RubyMetricsSchema = v.object({
|
|
236
|
+
/** Ruby size relative to the base characters. */
|
|
237
|
+
rubyScale: v.optional(v.pipe(v.number(), v.minValue(.2), v.maxValue(.8)), DEFAULTS.plan.metrics.rubyScale),
|
|
238
|
+
/** Space between the top of the em box and the bottom of the ruby. */
|
|
239
|
+
rubyGap: v.optional(v.pipe(v.number(), v.minValue(0), v.maxValue(.5)), DEFAULTS.plan.metrics.rubyGap),
|
|
240
|
+
/** Ruby is squeezed horizontally down to this scale before it may overhang. */
|
|
241
|
+
compressionFloor: v.optional(v.pipe(v.number(), v.minValue(.3), v.maxValue(1)), DEFAULTS.plan.metrics.compressionFloor),
|
|
242
|
+
/** Widest a ruby may get, overhanging its base symmetrically. */
|
|
243
|
+
maxRubyWidth: v.optional(v.pipe(v.number(), v.minValue(1), v.maxValue(3)), DEFAULTS.plan.metrics.maxRubyWidth)
|
|
244
|
+
});
|
|
245
|
+
const Count = v.pipe(v.number(), v.integer(), v.minValue(0));
|
|
246
|
+
const PlanOptionsSchema = v.object({
|
|
247
|
+
family: v.optional(v.pipe(v.string(), v.trim(), v.nonEmpty(), v.check((name) => !/source/i.test(name), "Source is a reserved font name of the base font")), DEFAULTS.plan.family),
|
|
248
|
+
rubyScript: v.optional(v.picklist(RUBY_SCRIPTS), DEFAULTS.plan.rubyScript),
|
|
249
|
+
/** The sources whose words get ruby. */
|
|
250
|
+
sources: v.optional(v.array(v.picklist(WORD_SOURCES)), () => [...WORD_SOURCES]),
|
|
251
|
+
/** The most common words left bare, by score: a reader knows them. Lone kanji count too. */
|
|
252
|
+
skip: v.optional(Count, DEFAULTS.plan.skip),
|
|
253
|
+
/** How many words get ruby after those skipped, by score; all of them when left out. */
|
|
254
|
+
take: v.optional(v.pipe(Count, v.minValue(1))),
|
|
255
|
+
/** Words on the JLPT lists of this level and every easier one stay bare. */
|
|
256
|
+
skipJlpt: v.optional(v.picklist(JLPT_LEVELS)),
|
|
257
|
+
/** Annotate a kanji alone in its run, or one no word claims after another kanji (a suffix); when off, only selectors give them ruby. */
|
|
258
|
+
isolated: v.optional(v.boolean(), DEFAULTS.plan.isolated),
|
|
259
|
+
/** A kanji alone in its run is annotated only when one reading has this share of the evidence. */
|
|
260
|
+
isolatedMinShare: v.optional(v.pipe(v.number(), v.minValue(.5), v.maxValue(1)), DEFAULTS.plan.isolatedMinShare),
|
|
261
|
+
words: v.optional(v.object({
|
|
262
|
+
/** Extra words in inline notation, `日[に]本[ほん]`. */
|
|
263
|
+
add: v.optional(v.array(v.string()), []),
|
|
264
|
+
/** Written forms never to annotate. */
|
|
265
|
+
remove: v.optional(v.array(v.string()), []),
|
|
266
|
+
/** Readings to make the default, in inline notation. */
|
|
267
|
+
pin: v.optional(v.array(v.string()), [])
|
|
268
|
+
}), {}),
|
|
269
|
+
metrics: v.optional(RubyMetricsSchema, {})
|
|
270
|
+
});
|
|
271
|
+
function resolveOptions(input = {}) {
|
|
272
|
+
return v.parse(PlanOptionsSchema, input);
|
|
273
|
+
}
|
|
274
|
+
const Codepoint = v.pipe(v.number(), v.integer(), v.minValue(0), v.maxValue(1114111));
|
|
275
|
+
/**
|
|
276
|
+
* The contract between the planner and the compiler. The compiler makes no
|
|
277
|
+
* decision about readings: it draws what `glyphs` describes and compiles `fea`.
|
|
278
|
+
* In `fea`, base characters are named `uniXXXX` / `uXXXXX` after their codepoint.
|
|
279
|
+
*/
|
|
280
|
+
const PlanSchema = v.object({
|
|
281
|
+
version: v.literal(1),
|
|
282
|
+
names: v.object({
|
|
283
|
+
family: v.string(),
|
|
284
|
+
description: v.string()
|
|
285
|
+
}),
|
|
286
|
+
metrics: v.object({
|
|
287
|
+
rubyScale: v.number(),
|
|
288
|
+
rubyGap: v.number(),
|
|
289
|
+
compressionFloor: v.number(),
|
|
290
|
+
maxRubyWidth: v.number()
|
|
291
|
+
}),
|
|
292
|
+
/** Base characters the rules name; each must be in the base font's cmap. */
|
|
293
|
+
codepoints: v.array(Codepoint),
|
|
294
|
+
/**
|
|
295
|
+
* Base + variation selector sequences to map in `cmap` format 14, each to an
|
|
296
|
+
* empty glyph named `name`, with the base's advance: the rules replace it.
|
|
297
|
+
*/
|
|
298
|
+
variants: v.array(v.object({
|
|
299
|
+
name: v.pipe(v.string(), v.regex(/^[A-Z_][\w.]{0,62}$/i)),
|
|
300
|
+
base: Codepoint,
|
|
301
|
+
selector: Codepoint
|
|
302
|
+
})),
|
|
303
|
+
glyphs: v.array(v.object({
|
|
304
|
+
name: v.pipe(v.string(), v.regex(/^[A-Z_][\w.]{0,62}$/i)),
|
|
305
|
+
/** The base characters drawn side by side; more than one makes a ligature. */
|
|
306
|
+
bases: v.pipe(v.array(Codepoint), v.minLength(1)),
|
|
307
|
+
/** Ruby text drawn above characters `[start, end)` of `bases`. */
|
|
308
|
+
ruby: v.array(v.object({
|
|
309
|
+
start: v.number(),
|
|
310
|
+
end: v.number(),
|
|
311
|
+
text: v.pipe(v.string(), v.nonEmpty())
|
|
312
|
+
}))
|
|
313
|
+
})),
|
|
314
|
+
fea: v.string(),
|
|
315
|
+
/** Subset the base font to these codepoints before building (live previews). */
|
|
316
|
+
subset: v.optional(v.array(Codepoint)),
|
|
317
|
+
/** Shaping checks the compiler runs on its output: `text` must shape to `glyphs`. */
|
|
318
|
+
smoke: v.array(v.object({
|
|
319
|
+
text: v.string(),
|
|
320
|
+
glyphs: v.array(v.string())
|
|
321
|
+
}))
|
|
322
|
+
});
|
|
323
|
+
/**
|
|
324
|
+
* The bounds of a reading's score, 0 to 100: above `tagged`, a word JMdict's
|
|
325
|
+
* lists tag, by its weight; above `used`, one Tatoeba's sentences use (20 and
|
|
326
|
+
* its uses, up to 50); above `listed`, one a manual list (15) or JMnedict (10)
|
|
327
|
+
* names; any other JMdict word scores `listed`.
|
|
328
|
+
*/
|
|
329
|
+
const SCORE = {
|
|
330
|
+
tagged: 50,
|
|
331
|
+
used: 20,
|
|
332
|
+
listed: 5
|
|
333
|
+
};
|
|
334
|
+
/** Score points per factor e of weight: a point is a twelfth, about 9%, of it. */
|
|
335
|
+
const POINTS = 12;
|
|
336
|
+
/**
|
|
337
|
+
* The score of a tagged reading of corpus weight `weight`, at most 1: 1 / its
|
|
338
|
+
* newspaper bucket, or a lone kanji's share of its uses. It is 100 for 1, 92
|
|
339
|
+
* for 1/2, 54 for 1/48; a weight too small to score above `tagged` scores 0.
|
|
340
|
+
*/
|
|
341
|
+
function scoreOf(weight) {
|
|
342
|
+
const score = Math.round(100 + POINTS * Math.log(weight));
|
|
343
|
+
return score > SCORE.tagged ? score : 0;
|
|
344
|
+
}
|
|
345
|
+
/** The corpus weight of a reading's score, which `scoreOf` gives; 0 for one no list tags. */
|
|
346
|
+
const weightOf = (score) => score > SCORE.tagged ? Math.exp((score - 100) / POINTS) : 0;
|
|
347
|
+
const isRecord = (value) => typeof value === "object" && value !== null && !Array.isArray(value);
|
|
348
|
+
const isReading = (value) => Array.isArray(value) && value.length === 2 && value.every(Number.isInteger) && value[0] >= 0 && value[0] <= 100 && value[1] > 0 && value[1] < 1 << SOURCES.length;
|
|
349
|
+
/** Readings by run or kanji, then by alignment or reading. Checked by hand: valibot takes seconds over the 400,000 readings of the dictionary. */
|
|
350
|
+
const ReadingsSchema = v.custom((input) => isRecord(input) && Object.values(input).every((readings) => isRecord(readings) && Object.values(readings).every(isReading)), "Expected readings by run, each [score, sources]");
|
|
351
|
+
/** The bit of each source, which the file writes for its readers. */
|
|
352
|
+
const SOURCE_BITS = Object.fromEntries(SOURCES.map((source) => [source, sourceBit(source)]));
|
|
353
|
+
/**
|
|
354
|
+
* The compiled dictionary, the contract between `@furifont/dictionary` and
|
|
355
|
+
* the planner: readings by the Han run a font sees, each with its score and
|
|
356
|
+
* sources. A run's readings come ranked, the default first. Run readings are
|
|
357
|
+
* JmdictFurigana alignments within the run.
|
|
358
|
+
*/
|
|
359
|
+
const DictionarySchema = v.object({
|
|
360
|
+
/** Source datasets and the version each was compiled from. */
|
|
361
|
+
versions: v.record(v.string(), v.string()),
|
|
362
|
+
/** The bit of each source in the readings' masks. */
|
|
363
|
+
sources: v.custom((input) => isRecord(input) && JSON.stringify(input) === JSON.stringify(SOURCE_BITS), "Expected the sources of this version of @furifont/core"),
|
|
364
|
+
/** Kanji to its on'yomi, in hiragana. */
|
|
365
|
+
onyomi: v.record(v.string(), v.array(v.string())),
|
|
366
|
+
/** Runs of two or more kanji, then their alignments: `{ 日本: { '0-1:にほん': [75, 61] } }`. */
|
|
367
|
+
runs: ReadingsSchema,
|
|
368
|
+
/** Kanji alone in a run, then their readings: `{ 来: { く: [98, 2101] } }`. */
|
|
369
|
+
kanji: ReadingsSchema,
|
|
370
|
+
/** Written forms never to annotate, such as 月, whose reading alone is a guess; users' removals add to them. */
|
|
371
|
+
bare: v.array(v.string())
|
|
372
|
+
});
|
|
373
|
+
/** The file `DictionarySchema` reads: an entry a line. */
|
|
374
|
+
function formatDictionary(dictionary) {
|
|
375
|
+
const lines = (table) => Object.entries(table).map(([key, value]) => `${JSON.stringify(key)}:${JSON.stringify(value)}`).join(",\n");
|
|
376
|
+
return `{
|
|
377
|
+
"versions":${JSON.stringify(dictionary.versions)},
|
|
378
|
+
"sources":${JSON.stringify(dictionary.sources)},
|
|
379
|
+
"bare":${JSON.stringify(dictionary.bare)},
|
|
380
|
+
"onyomi":{
|
|
381
|
+
${lines(dictionary.onyomi)}
|
|
382
|
+
},
|
|
383
|
+
"kanji":{
|
|
384
|
+
${lines(dictionary.kanji)}
|
|
385
|
+
},
|
|
386
|
+
"runs":{
|
|
387
|
+
${lines(dictionary.runs)}
|
|
388
|
+
}
|
|
389
|
+
}
|
|
390
|
+
`;
|
|
391
|
+
}
|
|
392
|
+
//#endregion
|
|
393
|
+
//#region src/fuse.ts
|
|
394
|
+
/** JMdict orders readings within a form; a run read inside a longer form ranks after the form's own readings. */
|
|
395
|
+
const OTHER_FORM_ORDER = 1e3;
|
|
396
|
+
const OKURIGANA = /[\u3041-\u3096]/;
|
|
397
|
+
/** Uses in Tatoeba's sentences that make a word one of the sentences'. */
|
|
398
|
+
const MIN_USES = 3;
|
|
399
|
+
/** With this many uses alone in the sentences, they decide how a lone kanji reads rather than the word lists. */
|
|
400
|
+
const LONE_MIN_USES = 20;
|
|
401
|
+
/** A tagged reading of a kanji that the sentences decide keeps this score at most, so it stays among the kanji's readings. */
|
|
402
|
+
const LONE_TAGGED_FLOOR = 60;
|
|
403
|
+
const PRIORITY_LISTS = [
|
|
404
|
+
"ichi",
|
|
405
|
+
"news",
|
|
406
|
+
"spec",
|
|
407
|
+
"gai"
|
|
408
|
+
];
|
|
409
|
+
const MANUAL_BITS = MANUAL_SOURCES.reduce((mask, name) => mask | sourceBit(name), 0);
|
|
410
|
+
/** The evidence of `text` read as `key`, met for the first time or not. */
|
|
411
|
+
function evidenceOf(table, text, key) {
|
|
412
|
+
return getOrCreate(getOrCreate(table, text, () => /* @__PURE__ */ new Map()), key, () => ({ sources: 0 }));
|
|
413
|
+
}
|
|
414
|
+
/** Spans as a JmdictFurigana alignment: `0:に;1:ほん`, `0-1:きょう`. */
|
|
415
|
+
function formatFurigana(spans) {
|
|
416
|
+
return spans.map((s) => `${s.end - s.start > 1 ? `${s.start}-${s.end - 1}` : s.start}:${s.ruby}`).join(";");
|
|
417
|
+
}
|
|
418
|
+
/** Rough corpus frequency of a tagged entry, in newspaper buckets of ~500 words each. */
|
|
419
|
+
function rankOf(entry) {
|
|
420
|
+
if (entry.priority) return entry.nf || Number(/nf(\d+)/.exec(entry.priority)?.[1] ?? 0) || (/[a-z]1\b/.test(entry.priority) ? 24 : 48);
|
|
421
|
+
}
|
|
422
|
+
/** Adds what a JMdict entry says of a reading: its lists, its rank and its JLPT level. */
|
|
423
|
+
function addEntry(into, entry) {
|
|
424
|
+
into.sources |= sourceBit("jmdict");
|
|
425
|
+
if (entry.jlpt) into.jlpt = Math.max(into.jlpt ?? 0, entry.jlpt);
|
|
426
|
+
const rank = rankOf(entry);
|
|
427
|
+
if (!rank) return;
|
|
428
|
+
into.rank = Math.min(into.rank ?? rank, rank);
|
|
429
|
+
if (entry.nf) into.sources |= sourceBit("nf");
|
|
430
|
+
for (const list of PRIORITY_LISTS.filter((list) => new RegExp(`\\b${list}[12]\\b`).test(entry.priority))) into.sources |= sourceBit(list);
|
|
431
|
+
}
|
|
432
|
+
function addRun(runs, run, entry) {
|
|
433
|
+
const evidence = evidenceOf(runs, run.text, formatFurigana(run.spans));
|
|
434
|
+
const order = run.text === entry.form ? entry.order : OTHER_FORM_ORDER + entry.order;
|
|
435
|
+
evidence.order = Math.min(evidence.order ?? order, order);
|
|
436
|
+
if (entry.tatoeba) evidence.uses = Math.max(evidence.uses ?? 0, entry.tatoeba);
|
|
437
|
+
addEntry(evidence, entry);
|
|
438
|
+
}
|
|
439
|
+
function addKanji(kanji, run, entry) {
|
|
440
|
+
const ruby = run.spans[0].ruby;
|
|
441
|
+
if (entry.tatoeba && !entry.conjugatesReading) {
|
|
442
|
+
const uses = getOrCreate(kanji.uses, run.text, () => /* @__PURE__ */ new Map());
|
|
443
|
+
uses.set(ruby, (uses.get(ruby) ?? 0) + entry.tatoeba);
|
|
444
|
+
}
|
|
445
|
+
if (rankOf(entry)) for (const r of entry.conjugatesReading && ruby === "く" ? [
|
|
446
|
+
"く",
|
|
447
|
+
"き",
|
|
448
|
+
"こ"
|
|
449
|
+
] : [ruby]) addEntry(evidenceOf(kanji.tagged, run.text, r), entry);
|
|
450
|
+
else if (OKURIGANA.test(entry.form[entry.form.indexOf(run.text) + run.text.length] ?? "")) getOrCreate(kanji.inflecting, run.text, () => /* @__PURE__ */ new Set()).add(ruby);
|
|
451
|
+
}
|
|
452
|
+
/** Its sources with the JLPT level and Tatoeba's sentences as bits. */
|
|
453
|
+
function sourcesOf(evidence) {
|
|
454
|
+
const level = evidence.jlpt && JLPT_LEVELS[5 - evidence.jlpt];
|
|
455
|
+
return evidence.sources | (level ? sourceBit(level) : 0) | ((evidence.uses ?? 0) >= MIN_USES ? sourceBit("tatoeba") : 0);
|
|
456
|
+
}
|
|
457
|
+
/** How common a run's reading is, by the most telling of its sources. */
|
|
458
|
+
function runScore(evidence) {
|
|
459
|
+
const uses = evidence.uses ?? 0;
|
|
460
|
+
return Math.max(evidence.rank ? scoreOf(1 / evidence.rank) : 0, uses >= MIN_USES ? SCORE.used + Math.min(uses, SCORE.tagged - SCORE.used) : 0, evidence.sources & MANUAL_BITS ? 15 : 0, evidence.sources & sourceBit("place") ? 10 : 0, evidence.order === void 0 ? 0 : SCORE.listed);
|
|
461
|
+
}
|
|
462
|
+
const tier = (score) => [
|
|
463
|
+
SCORE.tagged,
|
|
464
|
+
SCORE.used,
|
|
465
|
+
SCORE.listed
|
|
466
|
+
].filter((bound) => score > bound).length;
|
|
467
|
+
/**
|
|
468
|
+
* A run's readings ranked, the default first: tagged ones, then those the
|
|
469
|
+
* sentences use, each by JMdict's order; then the lists' and the place
|
|
470
|
+
* names', then the other JMdict words', each by score. Readings of the same
|
|
471
|
+
* kana are one, under the alignment that splits them most: 倉本 くら・もと
|
|
472
|
+
* rather than くらもと.
|
|
473
|
+
*/
|
|
474
|
+
function rankReadings(readings) {
|
|
475
|
+
const byKana = /* @__PURE__ */ new Map();
|
|
476
|
+
for (const [key, evidence] of readings) {
|
|
477
|
+
const spans = parseFurigana(key);
|
|
478
|
+
const reading = {
|
|
479
|
+
key,
|
|
480
|
+
spans,
|
|
481
|
+
score: runScore(evidence),
|
|
482
|
+
sources: sourcesOf(evidence),
|
|
483
|
+
order: evidence.order ?? Infinity
|
|
484
|
+
};
|
|
485
|
+
const kana = spans.map((s) => s.ruby).join("");
|
|
486
|
+
const same = byKana.get(kana);
|
|
487
|
+
if (!same) {
|
|
488
|
+
byKana.set(kana, reading);
|
|
489
|
+
continue;
|
|
490
|
+
}
|
|
491
|
+
const finer = reading.spans.length > same.spans.length ? reading : same;
|
|
492
|
+
byKana.set(kana, {
|
|
493
|
+
...finer,
|
|
494
|
+
score: Math.max(same.score, reading.score),
|
|
495
|
+
sources: same.sources | reading.sources,
|
|
496
|
+
order: Math.min(same.order, reading.order)
|
|
497
|
+
});
|
|
498
|
+
}
|
|
499
|
+
return [...byKana.values()].sort((a, b) => tier(b.score) - tier(a.score) || (tier(a.score) > 1 ? a.order - b.order || b.score - a.score : b.score - a.score || a.order - b.order)).map((r) => [r.key, [r.score, r.sources]]);
|
|
500
|
+
}
|
|
501
|
+
/**
|
|
502
|
+
* When the sentences use a kanji alone often enough to tell, each reading
|
|
503
|
+
* scores by its share of those uses, so that its weight in the planner is
|
|
504
|
+
* about that share; a tagged one keeps a little, to stay among its readings.
|
|
505
|
+
*/
|
|
506
|
+
function scoreUses(readings, uses) {
|
|
507
|
+
const total = [...uses.values()].reduce((a, b) => a + b, 0);
|
|
508
|
+
if (total < LONE_MIN_USES) return;
|
|
509
|
+
for (const reading of readings.values()) reading[0] = Math.min(reading[0], LONE_TAGGED_FLOOR);
|
|
510
|
+
for (const [ruby, n] of uses) {
|
|
511
|
+
const reading = getOrCreate(readings, ruby, () => [0, sourceBit("jmdict")]);
|
|
512
|
+
reading[0] = Math.max(reading[0], scoreOf(n / total));
|
|
513
|
+
if (n >= MIN_USES) reading[1] |= sourceBit("tatoeba");
|
|
514
|
+
}
|
|
515
|
+
}
|
|
516
|
+
/** A lone kanji's readings, the most common first: tagged words and the sentences score them; rare verbs add readings that score nothing. */
|
|
517
|
+
function kanjiReadings(kanji) {
|
|
518
|
+
const out = /* @__PURE__ */ new Map();
|
|
519
|
+
for (const [text, tagged] of kanji.tagged) {
|
|
520
|
+
const readings = new Map([...tagged].map(([ruby, evidence]) => [ruby, [scoreOf(1 / evidence.rank), sourcesOf(evidence)]]));
|
|
521
|
+
scoreUses(readings, kanji.uses.get(text) ?? /* @__PURE__ */ new Map());
|
|
522
|
+
out.set(text, new Map([...readings].filter(([, [score]]) => score).sort((a, b) => b[1][0] - a[1][0])));
|
|
523
|
+
}
|
|
524
|
+
for (const [text, rubies] of kanji.inflecting) {
|
|
525
|
+
const readings = getOrCreate(out, text, () => /* @__PURE__ */ new Map());
|
|
526
|
+
for (const ruby of rubies) {
|
|
527
|
+
const [score, sources] = readings.get(ruby) ?? [0, sourceBit("jmdict")];
|
|
528
|
+
readings.set(ruby, [score, sources | sourceBit("inflecting")]);
|
|
529
|
+
}
|
|
530
|
+
}
|
|
531
|
+
return out;
|
|
532
|
+
}
|
|
533
|
+
function notationRuns$1(notation) {
|
|
534
|
+
const { form, spans } = parseNotation(notation);
|
|
535
|
+
return hanRuns(form, spans);
|
|
536
|
+
}
|
|
537
|
+
/** The runs each list names, as the planner reads them: no lone kanji, which would be no evidence for the kanji's reading. */
|
|
538
|
+
function listedRuns(sources) {
|
|
539
|
+
const manual = MANUAL_SOURCES.flatMap((name) => (sources.manual[name] ?? []).flatMap(notationRuns$1).map((run) => [run, name]));
|
|
540
|
+
const places = sources.places.flatMap((place) => {
|
|
541
|
+
const runs = hanRuns(place.form, parseFurigana(place.furigana));
|
|
542
|
+
return runs.length === 1 && runs[0].text === place.form ? [[runs[0], "place"]] : [];
|
|
543
|
+
});
|
|
544
|
+
return [...manual, ...places].filter(([run]) => [...run.text].length > 1);
|
|
545
|
+
}
|
|
546
|
+
function addSource(readings, key, source) {
|
|
547
|
+
const [score, sources] = readings.get(key) ?? [0, 0];
|
|
548
|
+
readings.set(key, [score, sources | sourceBit(source)]);
|
|
549
|
+
}
|
|
550
|
+
/**
|
|
551
|
+
* Merges every source into the readings of the Han runs a font sees, each
|
|
552
|
+
* reading with its score and sources. What no plan can use is left out: a
|
|
553
|
+
* word's runs with an unannotated kanji, lone kanji of the lists or of words
|
|
554
|
+
* no list tags but before okurigana, and the on'yomi of kanji no reading uses.
|
|
555
|
+
*/
|
|
556
|
+
function fuseDictionary(sources) {
|
|
557
|
+
const runs = /* @__PURE__ */ new Map();
|
|
558
|
+
const kanji = {
|
|
559
|
+
tagged: /* @__PURE__ */ new Map(),
|
|
560
|
+
uses: /* @__PURE__ */ new Map(),
|
|
561
|
+
inflecting: /* @__PURE__ */ new Map()
|
|
562
|
+
};
|
|
563
|
+
for (const entry of sources.entries) for (const run of hanRuns(entry.form, parseFurigana(entry.furigana))) if ([...run.text].length > 1) addRun(runs, run, entry);
|
|
564
|
+
else addKanji(kanji, run, entry);
|
|
565
|
+
for (const [run, name] of listedRuns(sources)) evidenceOf(runs, run.text, formatFurigana(run.spans)).sources |= sourceBit(name);
|
|
566
|
+
const ranked = new Map([...runs].map(([text, readings]) => [text, new Map(rankReadings(readings))]));
|
|
567
|
+
const lone = kanjiReadings(kanji);
|
|
568
|
+
for (const run of sources.defaults.flatMap(notationRuns$1)) {
|
|
569
|
+
const [table, key] = [...run.text].length > 1 ? [ranked, formatFurigana(run.spans)] : [lone, run.spans[0].ruby];
|
|
570
|
+
addSource(getOrCreate(table, run.text, () => /* @__PURE__ */ new Map()), key, "default");
|
|
571
|
+
}
|
|
572
|
+
const used = new Set([...ranked.keys(), ...lone.keys()].flatMap((text) => [...text]));
|
|
573
|
+
const toRecord = (table) => Object.fromEntries([...table].map(([text, readings]) => [text, Object.fromEntries(readings)]));
|
|
574
|
+
return {
|
|
575
|
+
versions: sources.versions,
|
|
576
|
+
sources: SOURCE_BITS,
|
|
577
|
+
onyomi: Object.fromEntries(Object.entries(sources.onyomi).filter(([k]) => used.has(k))),
|
|
578
|
+
runs: toRecord(ranked),
|
|
579
|
+
kanji: toRecord(lone),
|
|
580
|
+
bare: sources.bare
|
|
581
|
+
};
|
|
582
|
+
}
|
|
181
583
|
//#endregion
|
|
182
584
|
//#region src/glyphs.ts
|
|
183
585
|
function glyphName(codepoint) {
|
|
@@ -528,94 +930,67 @@ const MIN_ALTERNATE_SHARE = .02;
|
|
|
528
930
|
/** A suffix reading is the one a kanji takes in this share of the runs it ends, out of at least `SUFFIX_MIN_RUNS`. */
|
|
529
931
|
const SUFFIX_MIN_SHARE = .9;
|
|
530
932
|
const SUFFIX_MIN_RUNS = 10;
|
|
531
|
-
/** With this many uses alone in the example sentences, they decide a lone kanji rather than the word lists. */
|
|
532
|
-
const LONE_MIN_USES = 20;
|
|
533
|
-
/** JMdict orders readings within a form; a run read inside a longer form ranks after the form's own readings. */
|
|
534
|
-
const OTHER_FORM_ORDER = 1e3;
|
|
535
933
|
const length = (text) => [...text].length;
|
|
536
934
|
function spanKey(spans) {
|
|
537
935
|
return spans.map((s) => `${s.start}-${s.end}:${s.ruby}`).join(";");
|
|
538
936
|
}
|
|
937
|
+
/** Pinned first; then readings that are evidence, before a place name's or a rare word's; then as the dictionary ranks them. */
|
|
539
938
|
function rank(a, b) {
|
|
540
|
-
return (b.pinned ?? 0) - (a.pinned ?? 0) || Number(
|
|
541
|
-
}
|
|
542
|
-
/** Rough corpus frequency of an entry: nf buckets hold ~500 words each, ranked by frequency. */
|
|
543
|
-
function entryWeight(entry) {
|
|
544
|
-
if (!entry.priority) return 0;
|
|
545
|
-
const nf = entry.nf || Number(/nf(\d+)/.exec(entry.priority)?.[1] ?? 0);
|
|
546
|
-
if (nf) return 1 / nf;
|
|
547
|
-
return /[a-z]1\b/.test(entry.priority) ? 1 / 24 : 1 / 48;
|
|
548
|
-
}
|
|
549
|
-
function keepBest(current, value) {
|
|
550
|
-
current.weight = Math.max(current.weight, value.weight);
|
|
551
|
-
if (value.nf && (!current.nf || value.nf < current.nf)) current.nf = value.nf;
|
|
552
|
-
current.jlpt = Math.max(current.jlpt, value.jlpt);
|
|
553
|
-
for (const tag of value.tags) current.tags.add(tag);
|
|
554
|
-
}
|
|
555
|
-
function priorityTags(entry) {
|
|
556
|
-
return new Set(entry.priority.split(" ").filter(Boolean));
|
|
557
|
-
}
|
|
558
|
-
function getOrCreate(map, key, create) {
|
|
559
|
-
let value = map.get(key);
|
|
560
|
-
if (value === void 0) map.set(key, value = create());
|
|
561
|
-
return value;
|
|
939
|
+
return (b.pinned ?? 0) - (a.pinned ?? 0) || Number(a.place ?? false) - Number(b.place ?? false) || a.order - b.order;
|
|
562
940
|
}
|
|
563
|
-
|
|
564
|
-
|
|
565
|
-
|
|
566
|
-
|
|
567
|
-
|
|
568
|
-
|
|
569
|
-
|
|
570
|
-
|
|
571
|
-
|
|
572
|
-
|
|
573
|
-
|
|
574
|
-
|
|
575
|
-
|
|
576
|
-
|
|
577
|
-
|
|
578
|
-
|
|
579
|
-
|
|
941
|
+
const isJmdict = (reading) => !!(reading.sources & sourceBit("jmdict"));
|
|
942
|
+
const PIN = sourceBit("default");
|
|
943
|
+
function indexRuns(index, runs) {
|
|
944
|
+
for (const [text, readings] of Object.entries(runs)) {
|
|
945
|
+
const ranked = [];
|
|
946
|
+
for (const [furigana, [score, sources]] of Object.entries(readings)) {
|
|
947
|
+
const spans = parseFurigana(furigana);
|
|
948
|
+
if (sources & PIN) index.defaults.push({
|
|
949
|
+
text,
|
|
950
|
+
spans
|
|
951
|
+
});
|
|
952
|
+
if (sources & ~PIN) ranked.push({
|
|
953
|
+
spans,
|
|
954
|
+
weight: weightOf(score),
|
|
955
|
+
score,
|
|
956
|
+
sources,
|
|
957
|
+
order: ranked.length
|
|
958
|
+
});
|
|
959
|
+
}
|
|
960
|
+
index.runs.set(text, ranked);
|
|
961
|
+
if (ranked.some(isJmdict)) index.jmdict.set(text, ranked.filter(isJmdict));
|
|
580
962
|
}
|
|
581
|
-
keepBest(current, reading);
|
|
582
|
-
current.tagged ||= reading.tagged;
|
|
583
|
-
current.order = Math.min(current.order, reading.order);
|
|
584
|
-
current.tatoeba = Math.max(current.tatoeba ?? 0, reading.tatoeba);
|
|
585
963
|
}
|
|
586
|
-
function
|
|
587
|
-
const
|
|
588
|
-
|
|
589
|
-
|
|
590
|
-
|
|
591
|
-
|
|
592
|
-
|
|
593
|
-
|
|
594
|
-
|
|
595
|
-
|
|
596
|
-
|
|
597
|
-
|
|
598
|
-
|
|
599
|
-
|
|
600
|
-
|
|
601
|
-
|
|
602
|
-
else readings.set(r, value);
|
|
964
|
+
function indexKanji(index, kanji) {
|
|
965
|
+
for (const [text, readings] of Object.entries(kanji)) for (const [ruby, [score, sources]] of Object.entries(readings)) {
|
|
966
|
+
if (weightOf(score)) getOrCreate(index.singles, text, () => /* @__PURE__ */ new Map()).set(ruby, {
|
|
967
|
+
weight: weightOf(score),
|
|
968
|
+
score,
|
|
969
|
+
sources
|
|
970
|
+
});
|
|
971
|
+
if (sources & sourceBit("inflecting")) getOrCreate(index.inflecting, text, () => /* @__PURE__ */ new Set()).add(ruby);
|
|
972
|
+
if (sources & PIN) index.defaults.push({
|
|
973
|
+
text,
|
|
974
|
+
spans: [{
|
|
975
|
+
start: 0,
|
|
976
|
+
end: 1,
|
|
977
|
+
ruby
|
|
978
|
+
}]
|
|
979
|
+
});
|
|
603
980
|
}
|
|
604
981
|
}
|
|
605
|
-
|
|
606
|
-
function
|
|
607
|
-
|
|
608
|
-
|
|
609
|
-
|
|
610
|
-
|
|
611
|
-
|
|
612
|
-
|
|
613
|
-
|
|
614
|
-
|
|
615
|
-
|
|
616
|
-
|
|
617
|
-
else if (OKURIGANA.test(entry.form[entry.form.indexOf(run.text) + run.text.length] ?? "")) getOrCreate(index.inflecting, run.text, () => /* @__PURE__ */ new Set()).add(run.spans[0].ruby);
|
|
618
|
-
}
|
|
982
|
+
/** The compiled dictionary as the planner looks it up. */
|
|
983
|
+
function indexDictionary(dictionary) {
|
|
984
|
+
const index = {
|
|
985
|
+
runs: /* @__PURE__ */ new Map(),
|
|
986
|
+
jmdict: /* @__PURE__ */ new Map(),
|
|
987
|
+
singles: /* @__PURE__ */ new Map(),
|
|
988
|
+
inflecting: /* @__PURE__ */ new Map(),
|
|
989
|
+
defaults: []
|
|
990
|
+
};
|
|
991
|
+
indexRuns(index, dictionary.runs);
|
|
992
|
+
indexKanji(index, dictionary.kanji);
|
|
993
|
+
return index;
|
|
619
994
|
}
|
|
620
995
|
function createContext(dictionary, options, coverage) {
|
|
621
996
|
const covered = (text) => codepoints(text).every((cp) => coverage.has(cp));
|
|
@@ -626,20 +1001,22 @@ function createContext(dictionary, options, coverage) {
|
|
|
626
1001
|
if (options.rubyScript === "katakanaOn" && kanji.length === 1 && isOnyomi(span.ruby, dictionary.onyomi[kanji[0]] ?? [])) return toKatakana(span.ruby);
|
|
627
1002
|
return span.ruby;
|
|
628
1003
|
};
|
|
629
|
-
const known = options.skipJlpt ?
|
|
1004
|
+
const known = options.skipJlpt ? JLPT_LEVELS.slice(0, JLPT_LEVELS.indexOf(options.skipJlpt) + 1).reduce((mask, level) => mask | sourceBit(level), 0) : 0;
|
|
630
1005
|
return {
|
|
631
1006
|
options,
|
|
632
|
-
removed: /* @__PURE__ */ new Set([...dictionary.bare
|
|
1007
|
+
removed: /* @__PURE__ */ new Set([...dictionary.bare, ...options.words.remove]),
|
|
633
1008
|
covered,
|
|
634
|
-
|
|
635
|
-
skipped: (r) => r.nf > 0 && r.nf < options.nfMin || r.jlpt >= known,
|
|
1009
|
+
known: (sources) => !!(sources & known),
|
|
636
1010
|
render,
|
|
637
1011
|
renderable: (run) => covered(run.text) && run.spans.every((_, i) => covered(render(run, i)))
|
|
638
1012
|
};
|
|
639
1013
|
}
|
|
640
|
-
function
|
|
1014
|
+
function notationRuns(notation) {
|
|
641
1015
|
const { form, spans } = parseNotation(notation);
|
|
642
|
-
|
|
1016
|
+
return hanRuns(form, spans);
|
|
1017
|
+
}
|
|
1018
|
+
function addCustomRuns(patterns, lonePins, runs, pinned, ctx) {
|
|
1019
|
+
for (const run of runs.filter(ctx.renderable)) {
|
|
643
1020
|
if (length(run.text) === 1) {
|
|
644
1021
|
if (pinned) lonePins.set(run.text, run.spans[0].ruby);
|
|
645
1022
|
continue;
|
|
@@ -651,65 +1028,69 @@ function addCustomWord(patterns, lonePins, notation, pinned, ctx) {
|
|
|
651
1028
|
else readings.push({
|
|
652
1029
|
spans: run.spans,
|
|
653
1030
|
weight: 1,
|
|
654
|
-
|
|
1031
|
+
score: 100,
|
|
1032
|
+
sources: 0,
|
|
655
1033
|
order: Number.MAX_SAFE_INTEGER,
|
|
656
|
-
nf: 1,
|
|
657
|
-
jlpt: 0,
|
|
658
|
-
tags: /* @__PURE__ */ new Set(),
|
|
659
1034
|
pinned
|
|
660
1035
|
});
|
|
661
1036
|
}
|
|
662
1037
|
}
|
|
663
|
-
/**
|
|
664
|
-
function
|
|
665
|
-
|
|
666
|
-
|
|
667
|
-
|
|
668
|
-
|
|
669
|
-
|
|
670
|
-
|
|
671
|
-
|
|
672
|
-
|
|
673
|
-
|
|
674
|
-
tagged: false,
|
|
675
|
-
order: 0,
|
|
676
|
-
nf: 0,
|
|
677
|
-
jlpt: 0,
|
|
678
|
-
tags: /* @__PURE__ */ new Set(),
|
|
679
|
-
place: true
|
|
680
|
-
}]);
|
|
681
|
-
}
|
|
1038
|
+
/** A run's readings from the plan's sources that the font can draw: those the lists or the sentences back, else the first, which is no evidence for its kanji's. */
|
|
1039
|
+
function wordReadings(text, readings, wanted, ctx) {
|
|
1040
|
+
const selected = readings.filter((r) => r.sources & wanted && ctx.renderable({
|
|
1041
|
+
text,
|
|
1042
|
+
spans: r.spans
|
|
1043
|
+
}));
|
|
1044
|
+
const backed = selected.filter((r) => r.score > SCORE.used);
|
|
1045
|
+
return backed.length ? backed.map((r) => ({ ...r })) : selected.slice(0, 1).map((r) => ({
|
|
1046
|
+
...r,
|
|
1047
|
+
place: true
|
|
1048
|
+
}));
|
|
682
1049
|
}
|
|
683
1050
|
/**
|
|
684
|
-
*
|
|
685
|
-
*
|
|
686
|
-
*
|
|
1051
|
+
* The words of the plan's sources, but those the reader knows, those the
|
|
1052
|
+
* user removes and, by score, the `skip` most common, words and lone kanji
|
|
1053
|
+
* together; then the `take` next words, and the user's own. A word is skipped
|
|
1054
|
+
* whole: were only its known reading dropped, 今日 would show こんにち.
|
|
687
1055
|
*/
|
|
688
|
-
function selectWords(index,
|
|
689
|
-
const
|
|
1056
|
+
function selectWords(index, ctx) {
|
|
1057
|
+
const wanted = ctx.options.sources.reduce((mask, source) => mask | sourceBit(source), 0);
|
|
690
1058
|
const skipped = /* @__PURE__ */ new Set();
|
|
1059
|
+
const skippedKanji = /* @__PURE__ */ new Set();
|
|
1060
|
+
const words = [];
|
|
691
1061
|
for (const [text, readings] of index.runs) {
|
|
692
1062
|
if (ctx.removed.has(text)) continue;
|
|
693
|
-
if (
|
|
1063
|
+
if (readings.some((r) => ctx.known(r.sources))) {
|
|
694
1064
|
skipped.add(text);
|
|
695
1065
|
continue;
|
|
696
1066
|
}
|
|
697
|
-
const
|
|
698
|
-
|
|
699
|
-
spans: r.spans
|
|
700
|
-
}));
|
|
701
|
-
if (common.length) patterns.set(text, common.map((r) => ({ ...r })));
|
|
1067
|
+
const chosen = wordReadings(text, readings, wanted, ctx);
|
|
1068
|
+
if (chosen.length) words.push([text, chosen]);
|
|
702
1069
|
}
|
|
703
|
-
|
|
1070
|
+
const kanji = [...index.singles].map(([text, readings]) => [text, [...readings.values()].reduce((a, b) => b.score > a.score ? b : a)]);
|
|
1071
|
+
for (const [text] of kanji.filter(([, top]) => ctx.known(top.sources))) skippedKanji.add(text);
|
|
1072
|
+
const byScore = [...words.map(([text, readings]) => ({
|
|
1073
|
+
text,
|
|
1074
|
+
score: Math.max(...readings.map((r) => r.score)),
|
|
1075
|
+
word: true
|
|
1076
|
+
})), ...kanji.filter(([text]) => !skippedKanji.has(text)).map(([text, top]) => ({
|
|
1077
|
+
text,
|
|
1078
|
+
score: top.score,
|
|
1079
|
+
word: false
|
|
1080
|
+
}))].sort((a, b) => b.score - a.score);
|
|
1081
|
+
for (const { text, word } of byScore.slice(0, ctx.options.skip)) (word ? skipped : skippedKanji).add(text);
|
|
1082
|
+
const taken = new Set(byScore.slice(ctx.options.skip).filter((item) => item.word).slice(0, ctx.options.take).map((item) => item.text));
|
|
1083
|
+
const patterns = new Map(words.filter(([text]) => taken.has(text)));
|
|
704
1084
|
const lonePins = /* @__PURE__ */ new Map();
|
|
705
|
-
|
|
706
|
-
|
|
707
|
-
|
|
1085
|
+
addCustomRuns(patterns, lonePins, ctx.options.words.add.flatMap(notationRuns), 0, ctx);
|
|
1086
|
+
addCustomRuns(patterns, lonePins, index.defaults, 1, ctx);
|
|
1087
|
+
addCustomRuns(patterns, lonePins, ctx.options.words.pin.flatMap(notationRuns), 2, ctx);
|
|
708
1088
|
for (const readings of patterns.values()) readings.sort(rank).splice(15);
|
|
709
1089
|
return {
|
|
710
1090
|
patterns,
|
|
711
1091
|
lonePins,
|
|
712
|
-
skipped
|
|
1092
|
+
skipped,
|
|
1093
|
+
skippedKanji
|
|
713
1094
|
};
|
|
714
1095
|
}
|
|
715
1096
|
/** Readings of a lone kanji the font can draw, heaviest first, with their share of the evidence. */
|
|
@@ -729,24 +1110,6 @@ function loneReadings(kanji, readings, ctx) {
|
|
|
729
1110
|
share: r.weight / total
|
|
730
1111
|
})).sort((a, b) => b.weight - a.weight);
|
|
731
1112
|
}
|
|
732
|
-
/** The reading running text gives a lone kanji most, with its share of the uses, when the sentences use the kanji alone often enough to tell. */
|
|
733
|
-
function usedReading(kanji, uses, ctx) {
|
|
734
|
-
const drawable = [...uses ?? []].filter(([ruby]) => ctx.renderable({
|
|
735
|
-
text: kanji,
|
|
736
|
-
spans: [{
|
|
737
|
-
start: 0,
|
|
738
|
-
end: 1,
|
|
739
|
-
ruby
|
|
740
|
-
}]
|
|
741
|
-
}));
|
|
742
|
-
const total = drawable.reduce((sum, [, n]) => sum + n, 0);
|
|
743
|
-
if (ctx.options.minTatoeba === 0 || total < LONE_MIN_USES) return;
|
|
744
|
-
const [ruby, n] = drawable.sort((a, b) => b[1] - a[1])[0];
|
|
745
|
-
return {
|
|
746
|
-
ruby,
|
|
747
|
-
share: n / total
|
|
748
|
-
};
|
|
749
|
-
}
|
|
750
1113
|
function addWordEvidence(patterns, add) {
|
|
751
1114
|
for (const [text, readings] of patterns) {
|
|
752
1115
|
const chars = [...text];
|
|
@@ -766,12 +1129,13 @@ function addRareVerbEvidence(inflecting, ctx, add) {
|
|
|
766
1129
|
}
|
|
767
1130
|
/**
|
|
768
1131
|
* A kanji alone in its run gets ruby only when one reading clearly dominates
|
|
769
|
-
* the common words it stands alone in,
|
|
1132
|
+
* the common words it stands alone in, or the sentences' uses of it alone
|
|
1133
|
+
* (ゴミ箱 is common, yet 箱 reads はこ), whatever the word list's size: verbs
|
|
770
1134
|
* like 書く rank low in newspapers yet decide how 書 reads. Every kanji also
|
|
771
1135
|
* gets an ordered list of its readings, which the variation selectors index
|
|
772
1136
|
* into, except a kanji whose dominant reading the reader is taken to know.
|
|
773
1137
|
*/
|
|
774
|
-
function chooseLoneKanji(index, patterns, lonePins, ctx) {
|
|
1138
|
+
function chooseLoneKanji(index, { patterns, lonePins, skippedKanji }, ctx) {
|
|
775
1139
|
const isolated = new Map(ctx.options.isolated ? lonePins : []);
|
|
776
1140
|
const skipped = /* @__PURE__ */ new Set();
|
|
777
1141
|
const evidence = /* @__PURE__ */ new Map();
|
|
@@ -784,9 +1148,8 @@ function chooseLoneKanji(index, patterns, lonePins, ctx) {
|
|
|
784
1148
|
if (ctx.removed.has(kanji)) continue;
|
|
785
1149
|
const ranked = loneReadings(kanji, readings, ctx).filter((r) => r.share >= MIN_ALTERNATE_SHARE);
|
|
786
1150
|
const top = ranked[0];
|
|
787
|
-
|
|
788
|
-
if (top &&
|
|
789
|
-
else if (chosen && chosen.share >= ctx.options.isolatedMinShare && ctx.options.isolated && !isolated.has(kanji)) isolated.set(kanji, chosen.ruby);
|
|
1151
|
+
if (top && !lonePins.has(kanji) && skippedKanji.has(kanji)) skipped.add(kanji);
|
|
1152
|
+
else if (top && top.share >= ctx.options.isolatedMinShare && ctx.options.isolated && !isolated.has(kanji)) isolated.set(kanji, top.ruby);
|
|
790
1153
|
for (const r of ranked) add(kanji, r.ruby, r.weight);
|
|
791
1154
|
}
|
|
792
1155
|
addWordEvidence(patterns, add);
|
|
@@ -817,11 +1180,11 @@ function chooseSuffixes(index, ctx) {
|
|
|
817
1180
|
const count = (counts, key) => void counts.set(key, (counts.get(key) ?? 0) + 1);
|
|
818
1181
|
const starts = /* @__PURE__ */ new Map();
|
|
819
1182
|
const ends = /* @__PURE__ */ new Map();
|
|
820
|
-
for (const [text, readings] of index.
|
|
1183
|
+
for (const [text, readings] of index.jmdict) {
|
|
821
1184
|
const chars = [...text];
|
|
822
1185
|
count(starts, chars[0]);
|
|
823
1186
|
count(ends, chars.at(-1));
|
|
824
|
-
for (const { spans } of readings
|
|
1187
|
+
for (const { spans } of readings) {
|
|
825
1188
|
const span = spans.at(-1);
|
|
826
1189
|
if (span.end - span.start === 1) count(getOrCreate(evidence, chars.at(-1), () => /* @__PURE__ */ new Map()), span.ruby);
|
|
827
1190
|
}
|
|
@@ -885,10 +1248,10 @@ function findMisreadRuns(index, patterns, kanjiReadings, suffixes, skipped, ctx)
|
|
|
885
1248
|
}
|
|
886
1249
|
const protect = (text, readings, longest) => {
|
|
887
1250
|
const chars = [...text];
|
|
888
|
-
const candidates = [...new Set(
|
|
889
|
-
const expected =
|
|
1251
|
+
const candidates = [...new Set(readings.map((r) => assignment(r.spans, chars.length).join("|")))];
|
|
1252
|
+
const expected = readings.map((r) => assignment(r.spans, chars.length));
|
|
890
1253
|
if (!expected.some((e) => misreads(chars, e, known, longest, suffixes))) return false;
|
|
891
|
-
const truth =
|
|
1254
|
+
const truth = readings.toSorted(rank)[0];
|
|
892
1255
|
const run = {
|
|
893
1256
|
text,
|
|
894
1257
|
spans: truth.spans
|
|
@@ -902,7 +1265,7 @@ function findMisreadRuns(index, patterns, kanjiReadings, suffixes, skipped, ctx)
|
|
|
902
1265
|
let longest = 2;
|
|
903
1266
|
for (const text of known.keys()) longest = Math.max(longest, length(text));
|
|
904
1267
|
changed = false;
|
|
905
|
-
for (const [text, readings] of index.
|
|
1268
|
+
for (const [text, readings] of index.jmdict) if (!known.has(text) && ctx.covered(text) && protect(text, readings, longest)) changed = true;
|
|
906
1269
|
}
|
|
907
1270
|
return protectedRuns;
|
|
908
1271
|
}
|
|
@@ -1027,7 +1390,7 @@ function buildRules(patterns, kanjiReadings, protectedRuns, glyphs) {
|
|
|
1027
1390
|
};
|
|
1028
1391
|
}
|
|
1029
1392
|
function smokeChecks(patterns, lone, glyphs) {
|
|
1030
|
-
const byFrequency = [...patterns].sort((a, b) =>
|
|
1393
|
+
const byFrequency = [...patterns].sort((a, b) => b[1][0].score - a[1][0].score);
|
|
1031
1394
|
const checks = [];
|
|
1032
1395
|
for (const grouped of [false, true]) for (const [text, readings] of byFrequency.filter(([text, r]) => isGrouped({
|
|
1033
1396
|
text,
|
|
@@ -1057,24 +1420,19 @@ function smokeChecks(patterns, lone, glyphs) {
|
|
|
1057
1420
|
return checks;
|
|
1058
1421
|
}
|
|
1059
1422
|
/**
|
|
1060
|
-
* Indexes the dictionary once
|
|
1423
|
+
* Indexes the compiled dictionary once, then plans fonts from it. Every
|
|
1061
1424
|
* decision about readings happens here: which words get ruby, which reading
|
|
1062
1425
|
* is the default, which longer words must be protected from a shorter match.
|
|
1063
1426
|
*/
|
|
1064
1427
|
function createFontPlanner(dictionary) {
|
|
1065
|
-
const index =
|
|
1066
|
-
|
|
1067
|
-
singles: /* @__PURE__ */ new Map(),
|
|
1068
|
-
inflecting: /* @__PURE__ */ new Map(),
|
|
1069
|
-
loneUses: /* @__PURE__ */ new Map()
|
|
1070
|
-
};
|
|
1071
|
-
for (const entry of dictionary.entries) indexEntry(index, entry);
|
|
1072
|
-
const sources = Object.entries(dictionary.sources).map(([name, version]) => `${name} ${version}`).join(", ");
|
|
1428
|
+
const index = indexDictionary(dictionary);
|
|
1429
|
+
const sources = Object.entries(dictionary.versions).map(([name, version]) => `${name} ${version}`).join(", ");
|
|
1073
1430
|
/** Every decision for one set of options, over the whole dictionary: slow, and the same whatever the preview text. */
|
|
1074
1431
|
function analyze(options, coverage) {
|
|
1075
1432
|
const ctx = createContext(dictionary, options, coverage);
|
|
1076
|
-
const
|
|
1077
|
-
const {
|
|
1433
|
+
const selection = selectWords(index, ctx);
|
|
1434
|
+
const { patterns, skipped } = selection;
|
|
1435
|
+
const { isolated, kanjiReadings } = chooseLoneKanji(index, selection, ctx);
|
|
1078
1436
|
const suffixes = chooseSuffixes(index, ctx);
|
|
1079
1437
|
const protectedRuns = findMisreadRuns(index, patterns, kanjiReadings, suffixes, skipped, ctx);
|
|
1080
1438
|
const glyphs = createGlyphRegistry(ctx.render);
|
|
@@ -1209,138 +1567,4 @@ function createFontPlanner(dictionary) {
|
|
|
1209
1567
|
};
|
|
1210
1568
|
}
|
|
1211
1569
|
//#endregion
|
|
1212
|
-
|
|
1213
|
-
const RUBY_SCRIPTS = [
|
|
1214
|
-
"hiragana",
|
|
1215
|
-
"katakanaOn",
|
|
1216
|
-
"hepburn"
|
|
1217
|
-
];
|
|
1218
|
-
const PRIORITY_TAGS = [
|
|
1219
|
-
"ichi1",
|
|
1220
|
-
"ichi2",
|
|
1221
|
-
"news1",
|
|
1222
|
-
"news2",
|
|
1223
|
-
"spec1",
|
|
1224
|
-
"spec2",
|
|
1225
|
-
"gai1",
|
|
1226
|
-
"gai2"
|
|
1227
|
-
];
|
|
1228
|
-
/** Easiest first. */
|
|
1229
|
-
const JLPT_LEVELS = [
|
|
1230
|
-
"N5",
|
|
1231
|
-
"N4",
|
|
1232
|
-
"N3",
|
|
1233
|
-
"N2",
|
|
1234
|
-
"N1"
|
|
1235
|
-
];
|
|
1236
|
-
/** Sizes in em, relative to the base font. */
|
|
1237
|
-
const RubyMetricsSchema = v.object({
|
|
1238
|
-
/** Ruby size relative to the base characters. */
|
|
1239
|
-
rubyScale: v.optional(v.pipe(v.number(), v.minValue(.2), v.maxValue(.8)), DEFAULTS.plan.metrics.rubyScale),
|
|
1240
|
-
/** Space between the top of the em box and the bottom of the ruby. */
|
|
1241
|
-
rubyGap: v.optional(v.pipe(v.number(), v.minValue(0), v.maxValue(.5)), DEFAULTS.plan.metrics.rubyGap),
|
|
1242
|
-
/** Ruby is squeezed horizontally down to this scale before it may overhang. */
|
|
1243
|
-
compressionFloor: v.optional(v.pipe(v.number(), v.minValue(.3), v.maxValue(1)), DEFAULTS.plan.metrics.compressionFloor),
|
|
1244
|
-
/** Widest a ruby may get, overhanging its base symmetrically. */
|
|
1245
|
-
maxRubyWidth: v.optional(v.pipe(v.number(), v.minValue(1), v.maxValue(3)), DEFAULTS.plan.metrics.maxRubyWidth)
|
|
1246
|
-
});
|
|
1247
|
-
const Bucket = v.pipe(v.number(), v.integer(), v.minValue(1), v.maxValue(48));
|
|
1248
|
-
/** `schema`, checking that its frequency window `nfMin..nfMax` holds a bucket; for schemas that extend the options too. */
|
|
1249
|
-
function withNfWindow(schema) {
|
|
1250
|
-
return v.pipe(schema, v.check((o) => o.nfMin <= o.nfMax, "nfMin must not exceed nfMax"));
|
|
1251
|
-
}
|
|
1252
|
-
const PlanOptionsSchema = withNfWindow(v.object({
|
|
1253
|
-
family: v.optional(v.pipe(v.string(), v.trim(), v.nonEmpty(), v.check((name) => !/source/i.test(name), "Source is a reserved font name of the base font")), DEFAULTS.plan.family),
|
|
1254
|
-
rubyScript: v.optional(v.picklist(RUBY_SCRIPTS), DEFAULTS.plan.rubyScript),
|
|
1255
|
-
/**
|
|
1256
|
-
* Words whose newspaper frequency bucket is below this stay bare: the most
|
|
1257
|
-
* common ones, which a reader knows. Tags don't bring them back.
|
|
1258
|
-
*/
|
|
1259
|
-
nfMin: v.optional(Bucket, DEFAULTS.plan.nfMin),
|
|
1260
|
-
/** Words whose newspaper frequency bucket is at most this are annotated. */
|
|
1261
|
-
nfMax: v.optional(Bucket, DEFAULTS.plan.nfMax),
|
|
1262
|
-
/**
|
|
1263
|
-
* JMdict priority tags that select a word whatever its frequency bucket:
|
|
1264
|
-
* `spec1` catches words like 日本 that the newspaper ranking misses.
|
|
1265
|
-
*/
|
|
1266
|
-
selectTags: v.optional(v.array(v.picklist(PRIORITY_TAGS)), () => [...DEFAULTS.plan.selectTags]),
|
|
1267
|
-
/**
|
|
1268
|
-
* Words of two or more kanji that Tatoeba's example sentences use at least
|
|
1269
|
-
* this many times are annotated whatever their bucket: everyday words that
|
|
1270
|
-
* no list tags, such as 留守中, which would otherwise be bare or read as a
|
|
1271
|
-
* place name (国中 くになか, not くにじゅう). The sentences also decide how a
|
|
1272
|
-
* lone kanji reads, where they use it alone often enough. 0 turns them off.
|
|
1273
|
-
*/
|
|
1274
|
-
minTatoeba: v.optional(v.pipe(v.number(), v.integer(), v.minValue(0)), DEFAULTS.plan.minTatoeba),
|
|
1275
|
-
/** Words on the JLPT lists of this level and every easier one stay bare. */
|
|
1276
|
-
skipJlpt: v.optional(v.picklist(JLPT_LEVELS)),
|
|
1277
|
-
/** Annotate place names from the dictionary where no common word claims the run. */
|
|
1278
|
-
places: v.optional(v.boolean(), DEFAULTS.plan.places),
|
|
1279
|
-
/** Annotate a kanji alone in its run, or one no word claims after another kanji (a suffix); when off, only selectors give them ruby. */
|
|
1280
|
-
isolated: v.optional(v.boolean(), DEFAULTS.plan.isolated),
|
|
1281
|
-
/** A kanji alone in its run is annotated only when one reading has this share of the evidence. */
|
|
1282
|
-
isolatedMinShare: v.optional(v.pipe(v.number(), v.minValue(.5), v.maxValue(1)), DEFAULTS.plan.isolatedMinShare),
|
|
1283
|
-
words: v.optional(v.object({
|
|
1284
|
-
/** Extra words in inline notation, `日[に]本[ほん]`. */
|
|
1285
|
-
add: v.optional(v.array(v.string()), []),
|
|
1286
|
-
/** Written forms never to annotate. */
|
|
1287
|
-
remove: v.optional(v.array(v.string()), []),
|
|
1288
|
-
/** Readings to make the default, in inline notation. */
|
|
1289
|
-
pin: v.optional(v.array(v.string()), [])
|
|
1290
|
-
}), {}),
|
|
1291
|
-
metrics: v.optional(RubyMetricsSchema, {})
|
|
1292
|
-
}));
|
|
1293
|
-
function resolveOptions(input = {}) {
|
|
1294
|
-
return v.parse(PlanOptionsSchema, input);
|
|
1295
|
-
}
|
|
1296
|
-
const Codepoint = v.pipe(v.number(), v.integer(), v.minValue(0), v.maxValue(1114111));
|
|
1297
|
-
/**
|
|
1298
|
-
* The contract between the planner and the compiler. The compiler makes no
|
|
1299
|
-
* decision about readings: it draws what `glyphs` describes and compiles `fea`.
|
|
1300
|
-
* In `fea`, base characters are named `uniXXXX` / `uXXXXX` after their codepoint.
|
|
1301
|
-
*/
|
|
1302
|
-
const PlanSchema = v.object({
|
|
1303
|
-
version: v.literal(1),
|
|
1304
|
-
names: v.object({
|
|
1305
|
-
family: v.string(),
|
|
1306
|
-
description: v.string()
|
|
1307
|
-
}),
|
|
1308
|
-
metrics: v.object({
|
|
1309
|
-
rubyScale: v.number(),
|
|
1310
|
-
rubyGap: v.number(),
|
|
1311
|
-
compressionFloor: v.number(),
|
|
1312
|
-
maxRubyWidth: v.number()
|
|
1313
|
-
}),
|
|
1314
|
-
/** Base characters the rules name; each must be in the base font's cmap. */
|
|
1315
|
-
codepoints: v.array(Codepoint),
|
|
1316
|
-
/**
|
|
1317
|
-
* Base + variation selector sequences to map in `cmap` format 14, each to an
|
|
1318
|
-
* empty glyph named `name`, with the base's advance: the rules replace it.
|
|
1319
|
-
*/
|
|
1320
|
-
variants: v.array(v.object({
|
|
1321
|
-
name: v.pipe(v.string(), v.regex(/^[A-Z_][\w.]{0,62}$/i)),
|
|
1322
|
-
base: Codepoint,
|
|
1323
|
-
selector: Codepoint
|
|
1324
|
-
})),
|
|
1325
|
-
glyphs: v.array(v.object({
|
|
1326
|
-
name: v.pipe(v.string(), v.regex(/^[A-Z_][\w.]{0,62}$/i)),
|
|
1327
|
-
/** The base characters drawn side by side; more than one makes a ligature. */
|
|
1328
|
-
bases: v.pipe(v.array(Codepoint), v.minLength(1)),
|
|
1329
|
-
/** Ruby text drawn above characters `[start, end)` of `bases`. */
|
|
1330
|
-
ruby: v.array(v.object({
|
|
1331
|
-
start: v.number(),
|
|
1332
|
-
end: v.number(),
|
|
1333
|
-
text: v.pipe(v.string(), v.nonEmpty())
|
|
1334
|
-
}))
|
|
1335
|
-
})),
|
|
1336
|
-
fea: v.string(),
|
|
1337
|
-
/** Subset the base font to these codepoints before building (live previews). */
|
|
1338
|
-
subset: v.optional(v.array(Codepoint)),
|
|
1339
|
-
/** Shaping checks the compiler runs on its output: `text` must shape to `glyphs`. */
|
|
1340
|
-
smoke: v.array(v.object({
|
|
1341
|
-
text: v.string(),
|
|
1342
|
-
glyphs: v.array(v.string())
|
|
1343
|
-
}))
|
|
1344
|
-
});
|
|
1345
|
-
//#endregion
|
|
1346
|
-
export { DEFAULTS, JLPT_LEVELS, MAX_READINGS, PREBUILT, PRIORITY_TAGS, PlanOptionsSchema, PlanSchema, RUBY_SCRIPTS, SELECTOR_FIRST_READING, SELECTOR_SUPPRESS, VERSION_PROBE, createFontPlanner, formatNotation, glyphName, hanRuns, isHan, isOnyomi, parseFurigana, parseNotation, resolveOptions, toHepburn, toHiragana, toKatakana, withNfWindow };
|
|
1570
|
+
export { DEFAULTS, DictionarySchema, JLPT_LEVELS, MANUAL_SOURCES, MAX_READINGS, PREBUILT, PlanOptionsSchema, PlanSchema, RUBY_SCRIPTS, SCORE, SELECTOR_FIRST_READING, SELECTOR_SUPPRESS, SOURCES, VERSION_PROBE, WORD_SOURCES, createFontPlanner, formatDictionary, formatFurigana, formatNotation, fuseDictionary, glyphName, hanRuns, isHan, isOnyomi, parseFurigana, parseNotation, resolveOptions, scoreOf, sourceBit, sourcesIn, toHepburn, toHiragana, toKatakana, weightOf };
|