@kensio/pinyinjs 1.6.0 → 1.7.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +211 -5
- package/data/core.entries +393 -392
- package/data/core.freq +0 -0
- package/data/core.keys +1 -0
- package/data/full.entries +10220 -10220
- package/data/manifest.json +12 -12
- package/data/script.map +7 -7
- package/data/standard.entries +957 -956
- package/data/standard.freq +0 -0
- package/data/standard.keys +1 -0
- package/dist/cli/arguments.d.ts +38 -0
- package/dist/cli/arguments.d.ts.map +1 -1
- package/dist/cli/arguments.js +46 -0
- package/dist/cli/arguments.js.map +1 -1
- package/dist/cli/commands.d.ts.map +1 -1
- package/dist/cli/commands.js +207 -1
- package/dist/cli/commands.js.map +1 -1
- package/dist/cli/help.d.ts.map +1 -1
- package/dist/cli/help.js +5 -0
- package/dist/cli/help.js.map +1 -1
- package/dist/decode/convert.d.ts +16 -0
- package/dist/decode/convert.d.ts.map +1 -1
- package/dist/decode/convert.js +3 -28
- package/dist/decode/convert.js.map +1 -1
- package/dist/decode/runs.d.ts +11 -0
- package/dist/decode/runs.d.ts.map +1 -1
- package/dist/decode/runs.js +19 -0
- package/dist/decode/runs.js.map +1 -1
- package/dist/decode/segment.d.ts +66 -0
- package/dist/decode/segment.d.ts.map +1 -0
- package/dist/decode/segment.js +79 -0
- package/dist/decode/segment.js.map +1 -0
- package/dist/dictionary/artifact.d.ts +10 -0
- package/dist/dictionary/artifact.d.ts.map +1 -1
- package/dist/dictionary/artifact.js +17 -0
- package/dist/dictionary/artifact.js.map +1 -1
- package/dist/dictionary/assertions.d.ts.map +1 -1
- package/dist/dictionary/assertions.js +38 -0
- package/dist/dictionary/assertions.js.map +1 -1
- package/dist/dictionary/corpus-mass.d.ts +68 -0
- package/dist/dictionary/corpus-mass.d.ts.map +1 -0
- package/dist/dictionary/corpus-mass.js +163 -0
- package/dist/dictionary/corpus-mass.js.map +1 -0
- package/dist/dictionary/dictionary.d.ts +56 -0
- package/dist/dictionary/dictionary.d.ts.map +1 -1
- package/dist/dictionary/dictionary.js +116 -1
- package/dist/dictionary/dictionary.js.map +1 -1
- package/dist/dictionary/erhua.d.ts +4 -0
- package/dist/dictionary/erhua.d.ts.map +1 -1
- package/dist/dictionary/erhua.js +6 -0
- package/dist/dictionary/erhua.js.map +1 -1
- package/dist/dictionary/merge.d.ts.map +1 -1
- package/dist/dictionary/merge.js +16 -8
- package/dist/dictionary/merge.js.map +1 -1
- package/dist/dictionary/tiers.d.ts +2 -1
- package/dist/dictionary/tiers.d.ts.map +1 -1
- package/dist/dictionary/tiers.js +77 -11
- package/dist/dictionary/tiers.js.map +1 -1
- package/dist/grade/check.d.ts +156 -0
- package/dist/grade/check.d.ts.map +1 -0
- package/dist/grade/check.js +356 -0
- package/dist/grade/check.js.map +1 -0
- package/dist/index.d.ts +11 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +5 -0
- package/dist/index.js.map +1 -1
- package/dist/search/candidates.d.ts +84 -0
- package/dist/search/candidates.d.ts.map +1 -0
- package/dist/search/candidates.js +290 -0
- package/dist/search/candidates.js.map +1 -0
- package/dist/search/match.d.ts +72 -0
- package/dist/search/match.d.ts.map +1 -0
- package/dist/search/match.js +419 -0
- package/dist/search/match.js.map +1 -0
- package/dist/search/query.d.ts +49 -0
- package/dist/search/query.d.ts.map +1 -0
- package/dist/search/query.js +115 -0
- package/dist/search/query.js.map +1 -0
- package/dist/search/reverse-index.d.ts +141 -0
- package/dist/search/reverse-index.d.ts.map +1 -0
- package/dist/search/reverse-index.js +356 -0
- package/dist/search/reverse-index.js.map +1 -0
- package/dist/sources/unihan.d.ts +9 -0
- package/dist/sources/unihan.d.ts.map +1 -1
- package/dist/sources/unihan.js +12 -1
- package/dist/sources/unihan.js.map +1 -1
- package/package.json +2 -3
- package/CHANGELOG.md +0 -634
|
@@ -0,0 +1,68 @@
|
|
|
1
|
+
import type { CedictEntry } from "../sources/cedict.js";
|
|
2
|
+
import type { JiebaEntry } from "../sources/jieba.js";
|
|
3
|
+
import { type ReadingField } from "../sources/unihan.js";
|
|
4
|
+
import type { Syllable } from "../syllable/syllable.js";
|
|
5
|
+
import type { TraditionalTable } from "./traditional.js";
|
|
6
|
+
/**
|
|
7
|
+
* The sources a character's corpus mass is counted over.
|
|
8
|
+
*
|
|
9
|
+
* The same three the merge already holds: the words, their readings, and how
|
|
10
|
+
* often jieba's corpus saw each word.
|
|
11
|
+
*/
|
|
12
|
+
export interface MassSources {
|
|
13
|
+
readonly phrase: ReadonlyMap<string, readonly string[]>;
|
|
14
|
+
readonly cedict: readonly CedictEntry[];
|
|
15
|
+
readonly jieba: ReadonlyMap<string, JiebaEntry>;
|
|
16
|
+
}
|
|
17
|
+
/**
|
|
18
|
+
* How much corpus each of a character's readings carries, keyed by reading.
|
|
19
|
+
*/
|
|
20
|
+
export type CorpusMass = ReadonlyMap<string, ReadonlyMap<string, number>>;
|
|
21
|
+
/**
|
|
22
|
+
* Count how much of jieba's corpus each reading of each character carries.
|
|
23
|
+
*
|
|
24
|
+
* Every word the dictionary knows votes for the reading it gives each of its
|
|
25
|
+
* characters, weighted by how often jieba's corpus saw that word. It is the one
|
|
26
|
+
* piece of evidence about a polyphone that is neither a lexicographer's pick nor
|
|
27
|
+
* a dictionary's page order: 行 is `xíng` because 进行, 行政 and 行为 outweigh
|
|
28
|
+
* 行业 and 银行, and nothing had to say so.
|
|
29
|
+
*
|
|
30
|
+
* **Single characters do not vote.** A one-character entry's reading *is* the
|
|
31
|
+
* default being ranked — it comes from Unihan, or from a source that picked one
|
|
32
|
+
* reading for the character — so counting it would have the ranking confirm
|
|
33
|
+
* itself. Only a word puts a character in a context that chose between its
|
|
34
|
+
* readings.
|
|
35
|
+
*
|
|
36
|
+
* **繁體 is counted through the script pairing.** jieba's corpus is 简体 and the
|
|
37
|
+
* phrase corpus is 简体, so a traditional character appears in none of the words
|
|
38
|
+
* counted here and would rank on nothing at all. Each vote is therefore also
|
|
39
|
+
* cast for whichever 繁體 form the word's reading picks — 髮 gets 头发's `fà` and
|
|
40
|
+
* 發 gets 发送's `fā` — which is the same evidence reaching the same character
|
|
41
|
+
* under the other script.
|
|
42
|
+
*/
|
|
43
|
+
export declare function countCorpusMass(sources: MassSources, traditional: TraditionalTable): CorpusMass;
|
|
44
|
+
/**
|
|
45
|
+
* Re-rank one character's readings by how much corpus each carries.
|
|
46
|
+
*
|
|
47
|
+
* The sort is stable and the mass of an unseen reading is zero, so a character
|
|
48
|
+
* no word covers keeps Unihan's order exactly, and so does the tail of readings
|
|
49
|
+
* below the ones words attest. The corpus only ever reorders what it has
|
|
50
|
+
* evidence about, and never past a leading 轻声 — see {@link rankingMass}.
|
|
51
|
+
*
|
|
52
|
+
* **Only where `kHanyuPinlu` did not already rank the character**, which is the
|
|
53
|
+
* whole of what this instrument is for. `kMandarin` names one reading and orders
|
|
54
|
+
* nothing, so where it decides the default there is nothing behind it, and that
|
|
55
|
+
* is what the corpus supplies. Where `kHanyuPinlu` lists several readings there
|
|
56
|
+
* already is a corpus ranking, and it is the better one: **this mass counts a
|
|
57
|
+
* character only inside words, so a reading used mostly as a bare character is
|
|
58
|
+
* invisible to it.** 呢 is 呢子 and 呢绒 in the dictionary and 呢 on the end of a
|
|
59
|
+
* question everywhere else; 都 is 首都 and 都市 against a bare 都 that is simply
|
|
60
|
+
* `dōu`; 将, 与, 处, 喝, 蒙 and 吗 are the same shape. `kHanyuPinlu` counts
|
|
61
|
+
* occurrences rather than words and has seen exactly what this cannot.
|
|
62
|
+
*
|
|
63
|
+
* Measured over CPP, that division is the whole difference between the rule
|
|
64
|
+
* paying and not: where `kHanyuPinlu` is silent or alone the mass fixes 438 and
|
|
65
|
+
* breaks 256, and where it ranks the character it fixes 201 and breaks 263.
|
|
66
|
+
*/
|
|
67
|
+
export declare function rankByCorpusMass(character: string, readings: readonly Syllable[], fields: ReadonlyMap<ReadingField, readonly string[]>, mass: CorpusMass): readonly Syllable[];
|
|
68
|
+
//# sourceMappingURL=corpus-mass.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"corpus-mass.d.ts","sourceRoot":"","sources":["../../src/dictionary/corpus-mass.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,sBAAsB,CAAC;AACxD,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,qBAAqB,CAAC;AACtD,OAAO,EAAqB,KAAK,YAAY,EAAE,MAAM,sBAAsB,CAAC;AAC5E,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,yBAAyB,CAAC;AAGxD,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,kBAAkB,CAAC;AAEzD;;;;;GAKG;AACH,MAAM,WAAW,WAAW;IAC1B,QAAQ,CAAC,MAAM,EAAE,WAAW,CAAC,MAAM,EAAE,SAAS,MAAM,EAAE,CAAC,CAAC;IACxD,QAAQ,CAAC,MAAM,EAAE,SAAS,WAAW,EAAE,CAAC;IACxC,QAAQ,CAAC,KAAK,EAAE,WAAW,CAAC,MAAM,EAAE,UAAU,CAAC,CAAC;CACjD;AAED;;GAEG;AACH,MAAM,MAAM,UAAU,GAAG,WAAW,CAAC,MAAM,EAAE,WAAW,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC;AAiC1E;;;;;;;;;;;;;;;;;;;;;GAqBG;AACH,wBAAgB,eAAe,CAC7B,OAAO,EAAE,WAAW,EACpB,WAAW,EAAE,gBAAgB,GAC5B,UAAU,CAyCZ;AA2CD;;;;;;;;;;;;;;;;;;;;;;GAsBG;AACH,wBAAgB,gBAAgB,CAC9B,SAAS,EAAE,MAAM,EACjB,QAAQ,EAAE,SAAS,QAAQ,EAAE,EAC7B,MAAM,EAAE,WAAW,CAAC,YAAY,EAAE,SAAS,MAAM,EAAE,CAAC,EACpD,IAAI,EAAE,UAAU,GACf,SAAS,QAAQ,EAAE,CAYrB"}
|
|
@@ -0,0 +1,163 @@
|
|
|
1
|
+
import { toCharacters } from "../script/characters.js";
|
|
2
|
+
import { isFrequencyRanked } from "../sources/unihan.js";
|
|
3
|
+
import { NEUTRAL_TONE } from "../tone/tone.js";
|
|
4
|
+
import { readAlignedReading } from "./reading.js";
|
|
5
|
+
/**
|
|
6
|
+
* A reading key that ignores 儿化, which is a suffix rather than a reading.
|
|
7
|
+
*/
|
|
8
|
+
function readingKey(syllable) {
|
|
9
|
+
return `${syllable.initial}|${syllable.final}|${String(syllable.tone)}`;
|
|
10
|
+
}
|
|
11
|
+
/**
|
|
12
|
+
* One reading per word, resolved the way the merge resolves it.
|
|
13
|
+
*
|
|
14
|
+
* The phrase corpus leads and CC-CEDICT fills its gaps, which is the precedence
|
|
15
|
+
* the merge itself applies. Taking both would let a word CC-CEDICT lists under
|
|
16
|
+
* several senses vote several times, and vote for readings the dictionary does
|
|
17
|
+
* not give it: 重点 is `zhòngdiǎn` in the dictionary, and counting every
|
|
18
|
+
* CC-CEDICT sense would have it also vote for `chóng`.
|
|
19
|
+
*/
|
|
20
|
+
function readingsByWord(sources) {
|
|
21
|
+
const readings = new Map();
|
|
22
|
+
for (const entry of sources.cedict) {
|
|
23
|
+
if (!readings.has(entry.simplified)) {
|
|
24
|
+
readings.set(entry.simplified, entry.readings);
|
|
25
|
+
}
|
|
26
|
+
}
|
|
27
|
+
for (const [word, reading] of sources.phrase) {
|
|
28
|
+
readings.set(word, reading);
|
|
29
|
+
}
|
|
30
|
+
return readings;
|
|
31
|
+
}
|
|
32
|
+
/**
|
|
33
|
+
* Count how much of jieba's corpus each reading of each character carries.
|
|
34
|
+
*
|
|
35
|
+
* Every word the dictionary knows votes for the reading it gives each of its
|
|
36
|
+
* characters, weighted by how often jieba's corpus saw that word. It is the one
|
|
37
|
+
* piece of evidence about a polyphone that is neither a lexicographer's pick nor
|
|
38
|
+
* a dictionary's page order: 行 is `xíng` because 进行, 行政 and 行为 outweigh
|
|
39
|
+
* 行业 and 银行, and nothing had to say so.
|
|
40
|
+
*
|
|
41
|
+
* **Single characters do not vote.** A one-character entry's reading *is* the
|
|
42
|
+
* default being ranked — it comes from Unihan, or from a source that picked one
|
|
43
|
+
* reading for the character — so counting it would have the ranking confirm
|
|
44
|
+
* itself. Only a word puts a character in a context that chose between its
|
|
45
|
+
* readings.
|
|
46
|
+
*
|
|
47
|
+
* **繁體 is counted through the script pairing.** jieba's corpus is 简体 and the
|
|
48
|
+
* phrase corpus is 简体, so a traditional character appears in none of the words
|
|
49
|
+
* counted here and would rank on nothing at all. Each vote is therefore also
|
|
50
|
+
* cast for whichever 繁體 form the word's reading picks — 髮 gets 头发's `fà` and
|
|
51
|
+
* 發 gets 发送's `fā` — which is the same evidence reaching the same character
|
|
52
|
+
* under the other script.
|
|
53
|
+
*/
|
|
54
|
+
export function countCorpusMass(sources, traditional) {
|
|
55
|
+
const mass = new Map();
|
|
56
|
+
const cast = (character, key, weight) => {
|
|
57
|
+
const byReading = mass.get(character) ?? new Map();
|
|
58
|
+
byReading.set(key, (byReading.get(key) ?? 0) + weight);
|
|
59
|
+
mass.set(character, byReading);
|
|
60
|
+
};
|
|
61
|
+
for (const [word, reading] of readingsByWord(sources)) {
|
|
62
|
+
const weight = sources.jieba.get(word)?.frequency ?? 0;
|
|
63
|
+
if (weight <= 0 || toCharacters(word).length < 2) {
|
|
64
|
+
continue;
|
|
65
|
+
}
|
|
66
|
+
const aligned = readAlignedReading(word, reading);
|
|
67
|
+
if (aligned === undefined) {
|
|
68
|
+
continue;
|
|
69
|
+
}
|
|
70
|
+
for (const read of aligned) {
|
|
71
|
+
// Punctuation is written and unread, and so votes for nothing.
|
|
72
|
+
if (read.syllable === undefined) {
|
|
73
|
+
continue;
|
|
74
|
+
}
|
|
75
|
+
// A syllable covering two characters — 儿化 — reads the first of them and
|
|
76
|
+
// says nothing about the second, which is the same rule the 繁體
|
|
77
|
+
// derivation applies to the same alignment.
|
|
78
|
+
const character = toCharacters(read.characters)[0];
|
|
79
|
+
/* c8 ignore next -- an aligned group always names at least one character */
|
|
80
|
+
if (character === undefined) {
|
|
81
|
+
continue;
|
|
82
|
+
}
|
|
83
|
+
const key = readingKey(read.syllable);
|
|
84
|
+
cast(character, key, weight);
|
|
85
|
+
const hant = traditional.convertCharacter(character, read.syllable);
|
|
86
|
+
if (hant !== character) {
|
|
87
|
+
cast(hant, key, weight);
|
|
88
|
+
}
|
|
89
|
+
}
|
|
90
|
+
}
|
|
91
|
+
return mass;
|
|
92
|
+
}
|
|
93
|
+
/**
|
|
94
|
+
* The mass each reading is ranked on, with a leading 轻声 held in place.
|
|
95
|
+
*
|
|
96
|
+
* A 轻声 the fields put ahead of every full-tone reading ranks on the mass of
|
|
97
|
+
* whichever reading carries the most, so the corpus can reorder the readings
|
|
98
|
+
* behind it but never take the default away from it.
|
|
99
|
+
*
|
|
100
|
+
* 轻声 is the unstressed reading — a 语气词, a suffix, the second half of a
|
|
101
|
+
* reduplication — and unstressed is precisely the use this mass cannot see. It
|
|
102
|
+
* counts a character only inside the words the dictionary holds, and a particle
|
|
103
|
+
* spends its life on the end of a sentence rather than inside a word: every
|
|
104
|
+
* occurrence of 吧 the corpus can reach is 酒吧, 网吧 or 吧台, so `bā` carries
|
|
105
|
+
* 1027 of it and the `ba` that ends four sentences in five carries 225. The
|
|
106
|
+
* sources are not silent about that reading — `kMandarin` names `ba` and
|
|
107
|
+
* `kHanyuPinlu` counted it 2073 times — they say it about the bare character,
|
|
108
|
+
* which is the one place a word cannot vote.
|
|
109
|
+
*
|
|
110
|
+
* Only a 轻声 the fields already lead with is held, and only against being
|
|
111
|
+
* displaced. Where the corpus reaches a neutral reading the fields rank lower
|
|
112
|
+
* it still promotes it — 蔔 takes 萝卜's `bo` over `bó` — and where they lead
|
|
113
|
+
* with a full tone it ranks the readings as it finds them: 哩 is `lī` in
|
|
114
|
+
* `kMandarin`, and the corpus is free to prefer the `lǐ` of 哩程 and 平方哩.
|
|
115
|
+
*/
|
|
116
|
+
function rankingMass(readings, massOf) {
|
|
117
|
+
const ranked = readings.map((syllable) => ({
|
|
118
|
+
syllable,
|
|
119
|
+
mass: massOf(syllable),
|
|
120
|
+
}));
|
|
121
|
+
const heaviest = Math.max(0, ...ranked.map((reading) => reading.mass));
|
|
122
|
+
for (const reading of ranked) {
|
|
123
|
+
if (reading.syllable.tone !== NEUTRAL_TONE) {
|
|
124
|
+
break;
|
|
125
|
+
}
|
|
126
|
+
reading.mass = heaviest;
|
|
127
|
+
}
|
|
128
|
+
return ranked;
|
|
129
|
+
}
|
|
130
|
+
/**
|
|
131
|
+
* Re-rank one character's readings by how much corpus each carries.
|
|
132
|
+
*
|
|
133
|
+
* The sort is stable and the mass of an unseen reading is zero, so a character
|
|
134
|
+
* no word covers keeps Unihan's order exactly, and so does the tail of readings
|
|
135
|
+
* below the ones words attest. The corpus only ever reorders what it has
|
|
136
|
+
* evidence about, and never past a leading 轻声 — see {@link rankingMass}.
|
|
137
|
+
*
|
|
138
|
+
* **Only where `kHanyuPinlu` did not already rank the character**, which is the
|
|
139
|
+
* whole of what this instrument is for. `kMandarin` names one reading and orders
|
|
140
|
+
* nothing, so where it decides the default there is nothing behind it, and that
|
|
141
|
+
* is what the corpus supplies. Where `kHanyuPinlu` lists several readings there
|
|
142
|
+
* already is a corpus ranking, and it is the better one: **this mass counts a
|
|
143
|
+
* character only inside words, so a reading used mostly as a bare character is
|
|
144
|
+
* invisible to it.** 呢 is 呢子 and 呢绒 in the dictionary and 呢 on the end of a
|
|
145
|
+
* question everywhere else; 都 is 首都 and 都市 against a bare 都 that is simply
|
|
146
|
+
* `dōu`; 将, 与, 处, 喝, 蒙 and 吗 are the same shape. `kHanyuPinlu` counts
|
|
147
|
+
* occurrences rather than words and has seen exactly what this cannot.
|
|
148
|
+
*
|
|
149
|
+
* Measured over CPP, that division is the whole difference between the rule
|
|
150
|
+
* paying and not: where `kHanyuPinlu` is silent or alone the mass fixes 438 and
|
|
151
|
+
* breaks 256, and where it ranks the character it fixes 201 and breaks 263.
|
|
152
|
+
*/
|
|
153
|
+
export function rankByCorpusMass(character, readings, fields, mass) {
|
|
154
|
+
const byReading = mass.get(character);
|
|
155
|
+
if (byReading === undefined || isFrequencyRanked(fields)) {
|
|
156
|
+
return readings;
|
|
157
|
+
}
|
|
158
|
+
const ranking = rankingMass(readings, (syllable) => byReading.get(readingKey(syllable)) ?? 0);
|
|
159
|
+
return ranking
|
|
160
|
+
.toSorted((left, right) => right.mass - left.mass)
|
|
161
|
+
.map((ranked) => ranked.syllable);
|
|
162
|
+
}
|
|
163
|
+
//# sourceMappingURL=corpus-mass.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"corpus-mass.js","sourceRoot":"","sources":["../../src/dictionary/corpus-mass.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,yBAAyB,CAAC;AAGvD,OAAO,EAAE,iBAAiB,EAAqB,MAAM,sBAAsB,CAAC;AAE5E,OAAO,EAAE,YAAY,EAAE,MAAM,iBAAiB,CAAC;AAC/C,OAAO,EAAE,kBAAkB,EAAE,MAAM,cAAc,CAAC;AAoBlD;;GAEG;AACH,SAAS,UAAU,CAAC,QAAkB;IACpC,OAAO,GAAG,QAAQ,CAAC,OAAO,IAAI,QAAQ,CAAC,KAAK,IAAI,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;AAC1E,CAAC;AAED;;;;;;;;GAQG;AACH,SAAS,cAAc,CACrB,OAAoB;IAEpB,MAAM,QAAQ,GAAG,IAAI,GAAG,EAA6B,CAAC;IACtD,KAAK,MAAM,KAAK,IAAI,OAAO,CAAC,MAAM,EAAE,CAAC;QACnC,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,KAAK,CAAC,UAAU,CAAC,EAAE,CAAC;YACpC,QAAQ,CAAC,GAAG,CAAC,KAAK,CAAC,UAAU,EAAE,KAAK,CAAC,QAAQ,CAAC,CAAC;QACjD,CAAC;IACH,CAAC;IACD,KAAK,MAAM,CAAC,IAAI,EAAE,OAAO,CAAC,IAAI,OAAO,CAAC,MAAM,EAAE,CAAC;QAC7C,QAAQ,CAAC,GAAG,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;IAC9B,CAAC;IACD,OAAO,QAAQ,CAAC;AAClB,CAAC;AAED;;;;;;;;;;;;;;;;;;;;;GAqBG;AACH,MAAM,UAAU,eAAe,CAC7B,OAAoB,EACpB,WAA6B;IAE7B,MAAM,IAAI,GAAG,IAAI,GAAG,EAA+B,CAAC;IAEpD,MAAM,IAAI,GAAG,CAAC,SAAiB,EAAE,GAAW,EAAE,MAAc,EAAQ,EAAE;QACpE,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,CAAC,SAAS,CAAC,IAAI,IAAI,GAAG,EAAkB,CAAC;QACnE,SAAS,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,GAAG,MAAM,CAAC,CAAC;QACvD,IAAI,CAAC,GAAG,CAAC,SAAS,EAAE,SAAS,CAAC,CAAC;IACjC,CAAC,CAAC;IAEF,KAAK,MAAM,CAAC,IAAI,EAAE,OAAO,CAAC,IAAI,cAAc,CAAC,OAAO,CAAC,EAAE,CAAC;QACtD,MAAM,MAAM,GAAG,OAAO,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,SAAS,IAAI,CAAC,CAAC;QACvD,IAAI,MAAM,IAAI,CAAC,IAAI,YAAY,CAAC,IAAI,CAAC,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACjD,SAAS;QACX,CAAC;QACD,MAAM,OAAO,GAAG,kBAAkB,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;QAClD,IAAI,OAAO,KAAK,SAAS,EAAE,CAAC;YAC1B,SAAS;QACX,CAAC;QACD,KAAK,MAAM,IAAI,IAAI,OAAO,EAAE,CAAC;YAC3B,+DAA+D;YAC/D,IAAI,IAAI,CAAC,QAAQ,KAAK,SAAS,EAAE,CAAC;gBAChC,SAAS;YACX,CAAC;YACD,wEAAwE;YACxE,+DAA+D;YAC/D,4CAA4C;YAC5C,MAAM,SAAS,GAAG,YAAY,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,CAAC;YACnD,4EAA4E;YAC5E,IAAI,SAAS,KAAK,SAAS,EAAE,CAAC;gBAC5B,SAAS;YACX,CAAC;YACD,MAAM,GAAG,GAAG,UAAU,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;YACtC,IAAI,CAAC,SAAS,EAAE,GAAG,EAAE,MAAM,CAAC,CAAC;YAC7B,MAAM,IAAI,GAAG,WAAW,CAAC,gBAAgB,CAAC,SAAS,EAAE,IAAI,CAAC,QAAQ,CAAC,CAAC;YACpE,IAAI,IAAI,KAAK,SAAS,EAAE,CAAC;gBACvB,IAAI,CAAC,IAAI,EAAE,GAAG,EAAE,MAAM,CAAC,CAAC;YAC1B,CAAC;QACH,CAAC;IACH,CAAC;IAED,OAAO,IAAI,CAAC;AACd,CAAC;AAED;;;;;;;;;;;;;;;;;;;;;;GAsBG;AACH,SAAS,WAAW,CAClB,QAA6B,EAC7B,MAAsC;IAEtC,MAAM,MAAM,GAAG,QAAQ,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;QACzC,QAAQ;QACR,IAAI,EAAE,MAAM,CAAC,QAAQ,CAAC;KACvB,CAAC,CAAC,CAAC;IACJ,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC;IACvE,KAAK,MAAM,OAAO,IAAI,MAAM,EAAE,CAAC;QAC7B,IAAI,OAAO,CAAC,QAAQ,CAAC,IAAI,KAAK,YAAY,EAAE,CAAC;YAC3C,MAAM;QACR,CAAC;QACD,OAAO,CAAC,IAAI,GAAG,QAAQ,CAAC;IAC1B,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC;AAED;;;;;;;;;;;;;;;;;;;;;;GAsBG;AACH,MAAM,UAAU,gBAAgB,CAC9B,SAAiB,EACjB,QAA6B,EAC7B,MAAoD,EACpD,IAAgB;IAEhB,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;IACtC,IAAI,SAAS,KAAK,SAAS,IAAI,iBAAiB,CAAC,MAAM,CAAC,EAAE,CAAC;QACzD,OAAO,QAAQ,CAAC;IAClB,CAAC;IACD,MAAM,OAAO,GAAG,WAAW,CACzB,QAAQ,EACR,CAAC,QAAQ,EAAE,EAAE,CAAC,SAAS,CAAC,GAAG,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC,IAAI,CAAC,CACvD,CAAC;IACF,OAAO,OAAO;SACX,QAAQ,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC;SACjD,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;AACtC,CAAC"}
|
|
@@ -1,5 +1,28 @@
|
|
|
1
1
|
import type { Syllable } from "../syllable/syllable.js";
|
|
2
2
|
import { type DictionaryArtifact } from "./artifact.js";
|
|
3
|
+
/**
|
|
4
|
+
* Every key's reading, in key order, at the string level.
|
|
5
|
+
*
|
|
6
|
+
* The seam a second index over the same artifact is built through. A reading is
|
|
7
|
+
* `yin2 hang2` on the way in and on the way out, and no {@link Syllable} is
|
|
8
|
+
* constructed anywhere along it — which is what makes a pass over all 723,147
|
|
9
|
+
* keys affordable at load time rather than a decode of the whole dictionary.
|
|
10
|
+
*
|
|
11
|
+
* Handed out as a cursor rather than as an array because the array is 39 MB on
|
|
12
|
+
* the full tier. What the cursor holds instead is the character defaults a
|
|
13
|
+
* derived reading is assembled from, and dropping it releases them.
|
|
14
|
+
*/
|
|
15
|
+
export interface DictionaryReadings {
|
|
16
|
+
/** How many positions there are, which is the dictionary's own size. */
|
|
17
|
+
readonly size: number;
|
|
18
|
+
/**
|
|
19
|
+
* The reading at a position, in the artifact's tone-numbered spelling.
|
|
20
|
+
*
|
|
21
|
+
* Empty where the position is out of range, or where a character the word is
|
|
22
|
+
* written with is not itself a key.
|
|
23
|
+
*/
|
|
24
|
+
readonly readingAt: (at: number) => string;
|
|
25
|
+
}
|
|
3
26
|
/**
|
|
4
27
|
* What the dictionary knows about one word.
|
|
5
28
|
*/
|
|
@@ -60,6 +83,33 @@ export declare class Dictionary {
|
|
|
60
83
|
* How many words the dictionary holds, counting each script's key.
|
|
61
84
|
*/
|
|
62
85
|
get size(): number;
|
|
86
|
+
/**
|
|
87
|
+
* The word at a position, or the empty string where there is none.
|
|
88
|
+
*
|
|
89
|
+
* Positions are what a second index over the dictionary stores, since a
|
|
90
|
+
* position is one number where a word is a string — the reverse index in
|
|
91
|
+
* `src/search/` is built on that. The same position indexes
|
|
92
|
+
* {@link Dictionary.frequencyAt} and {@link DictionaryReadings.readingAt}.
|
|
93
|
+
*/
|
|
94
|
+
wordAt(at: number): string;
|
|
95
|
+
/**
|
|
96
|
+
* The frequency bucket of the word at a position, 0 rarest to 15 commonest.
|
|
97
|
+
*
|
|
98
|
+
* The bucket rather than {@link WordEntry.cost}, because ranking a candidate
|
|
99
|
+
* list wants the sixteen-value scale a counting sort runs on — and because
|
|
100
|
+
* reading it costs no decode, where asking for the entry would decode every
|
|
101
|
+
* candidate to learn one number. See {@link FrequencyTable.bucketOf}.
|
|
102
|
+
*/
|
|
103
|
+
frequencyAt(at: number): number;
|
|
104
|
+
/**
|
|
105
|
+
* A cursor over every key's reading, in key order.
|
|
106
|
+
*
|
|
107
|
+
* Two passes, mirroring `readArtifact`: the character defaults are gathered
|
|
108
|
+
* first, because 83.25% of the full tier's keys store no reading and mean
|
|
109
|
+
* "the characters' defaults, in order". Holding the cursor holds those
|
|
110
|
+
* defaults, so build what needs it, use it, and let it go.
|
|
111
|
+
*/
|
|
112
|
+
readingsInOrder(): DictionaryReadings;
|
|
63
113
|
/**
|
|
64
114
|
* Whether any word begins with this text. An exact match counts.
|
|
65
115
|
*
|
|
@@ -81,6 +131,12 @@ export declare class Dictionary {
|
|
|
81
131
|
*
|
|
82
132
|
* The lattice needs these for its single-character fallback edges: 行 has to
|
|
83
133
|
* offer `xíng`, `háng` and `héng` at a position no word covers.
|
|
134
|
+
*
|
|
135
|
+
* The alternates are read off the canonical form's line, the same form
|
|
136
|
+
* {@link Dictionary.lookup} found the entry under. Searching the index for
|
|
137
|
+
* the raw character instead would land on the insertion point for a Hong Kong
|
|
138
|
+
* glyph form that is not itself a key — 裏 sits between whatever keys
|
|
139
|
+
* surround it — and take some unrelated entry's alternates.
|
|
84
140
|
*/
|
|
85
141
|
readingsOf(character: string): readonly (readonly Syllable[])[];
|
|
86
142
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"dictionary.d.ts","sourceRoot":"","sources":["../../src/dictionary/dictionary.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,yBAAyB,CAAC;AACxD,OAAO,EAAiB,KAAK,kBAAkB,EAAE,MAAM,eAAe,CAAC;
|
|
1
|
+
{"version":3,"file":"dictionary.d.ts","sourceRoot":"","sources":["../../src/dictionary/dictionary.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,yBAAyB,CAAC;AACxD,OAAO,EAAiB,KAAK,kBAAkB,EAAE,MAAM,eAAe,CAAC;AA0EvE;;;;;;;;;;;GAWG;AACH,MAAM,WAAW,kBAAkB;IACjC,wEAAwE;IACxE,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB;;;;;OAKG;IACH,QAAQ,CAAC,SAAS,EAAE,CAAC,EAAE,EAAE,MAAM,KAAK,MAAM,CAAC;CAC5C;AAED;;GAEG;AACH,MAAM,WAAW,SAAS;IACxB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,mBAAmB;IACnB,QAAQ,CAAC,OAAO,EAAE,SAAS,QAAQ,EAAE,CAAC;IACtC,kDAAkD;IAClD,QAAQ,CAAC,aAAa,CAAC,EAAE,SAAS,QAAQ,EAAE,CAAC;IAC7C,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;IAC9B,QAAQ,CAAC,YAAY,EAAE,OAAO,CAAC;IAC/B;;;;;;;OAOG;IACH,QAAQ,CAAC,cAAc,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;IAC5C;;;;OAIG;IACH,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;CACvB;AAED;;;;;;;;;;;;;;;;;;GAkBG;AACH,qBAAa,UAAU;;IACrB;;OAEG;IACH,MAAM,CAAC,IAAI,CAAC,QAAQ,EAAE,kBAAkB,GAAG,UAAU,CAEpD;IAQD;;OAEG;IACH,OAAO,eAiBN;IA+ED;;OAEG;IACH,IAAI,IAAI,IAAI,MAAM,CAEjB;IAED;;;;;;;OAOG;IACH,MAAM,CAAC,EAAE,EAAE,MAAM,GAAG,MAAM,CAEzB;IAED;;;;;;;OAOG;IACH,WAAW,CAAC,EAAE,EAAE,MAAM,GAAG,MAAM,CAE9B;IAED;;;;;;;OAOG;IACH,eAAe,IAAI,kBAAkB,CAmBpC;IAED;;;;;OAKG;IACH,SAAS,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAE/B;IAED;;;;;;;OAOG;IACH,MAAM,CAAC,IAAI,EAAE,MAAM,GAAG,SAAS,GAAG,SAAS,CAa1C;IAED;;;;;;;;;;;OAWG;IACH,UAAU,CAAC,SAAS,EAAE,MAAM,GAAG,SAAS,CAAC,SAAS,QAAQ,EAAE,CAAC,EAAE,CAW9D;CACF"}
|
|
@@ -19,6 +19,49 @@ const PROPER_NOUN_FLAG = "p";
|
|
|
19
19
|
* for each doubling of the input, 39µs to 607µs.
|
|
20
20
|
*/
|
|
21
21
|
const NAME_BOUNDARIES = /^p(?<at>\d+(?:\.\d+)*)$/u;
|
|
22
|
+
/**
|
|
23
|
+
* The lowest code unit that begins a surrogate pair.
|
|
24
|
+
*/
|
|
25
|
+
const HIGH_SURROGATE = 0x1_00_00;
|
|
26
|
+
/**
|
|
27
|
+
* Whether a key is written with exactly one character, without allocating.
|
|
28
|
+
*
|
|
29
|
+
* Asked of every key in the dictionary when {@link Dictionary.readingsInOrder}
|
|
30
|
+
* gathers the character defaults, which is where splitting each key into
|
|
31
|
+
* characters would cost an array per key for an answer that is a length check.
|
|
32
|
+
*/
|
|
33
|
+
function isOneCharacter(key) {
|
|
34
|
+
if (key.length === 1) {
|
|
35
|
+
return true;
|
|
36
|
+
}
|
|
37
|
+
return key.length === 2 && (key.codePointAt(0) ?? 0) >= HIGH_SURROGATE;
|
|
38
|
+
}
|
|
39
|
+
/**
|
|
40
|
+
* A word's reading, joined from its characters' default readings.
|
|
41
|
+
*
|
|
42
|
+
* Walked by code point rather than split into characters first, for the same
|
|
43
|
+
* reason as {@link isOneCharacter}: this runs over every key in the dictionary
|
|
44
|
+
* three times during a reverse index build, and the array `toCharacters`
|
|
45
|
+
* allocates per key is 20% of it.
|
|
46
|
+
*
|
|
47
|
+
* Returns the empty string where any character has no default, which is what a
|
|
48
|
+
* tier missing a character would look like. No tier does: `selectTier` never
|
|
49
|
+
* drops a character some word is written with.
|
|
50
|
+
*/
|
|
51
|
+
function derivedReading(key, defaults) {
|
|
52
|
+
let joined = "";
|
|
53
|
+
let at = 0;
|
|
54
|
+
while (at < key.length) {
|
|
55
|
+
const width = (key.codePointAt(at) ?? 0) >= HIGH_SURROGATE ? 2 : 1;
|
|
56
|
+
const value = defaults.get(key.slice(at, at + width));
|
|
57
|
+
if (value === undefined) {
|
|
58
|
+
return "";
|
|
59
|
+
}
|
|
60
|
+
joined = joined === "" ? value : `${joined} ${value}`;
|
|
61
|
+
at += width;
|
|
62
|
+
}
|
|
63
|
+
return joined;
|
|
64
|
+
}
|
|
22
65
|
/**
|
|
23
66
|
* A compiled dictionary, queried without being parsed.
|
|
24
67
|
*
|
|
@@ -114,12 +157,78 @@ export class Dictionary {
|
|
|
114
157
|
cost: this.#frequencies.costOf(at),
|
|
115
158
|
};
|
|
116
159
|
}
|
|
160
|
+
/**
|
|
161
|
+
* The reading column of a line, without slicing or splitting the rest of it.
|
|
162
|
+
*
|
|
163
|
+
* Deriving a word's reading needs its characters' readings and nothing else
|
|
164
|
+
* about them, and this is asked of every line in the dictionary during a
|
|
165
|
+
* reverse index build, so the other four columns are never touched.
|
|
166
|
+
*/
|
|
167
|
+
#readingColumn(at) {
|
|
168
|
+
const start = this.#lineStarts[at];
|
|
169
|
+
const nextStart = this.#lineStarts[at + 1];
|
|
170
|
+
if (start === undefined || nextStart === undefined) {
|
|
171
|
+
return "";
|
|
172
|
+
}
|
|
173
|
+
const end = nextStart - 1;
|
|
174
|
+
const column = this.#entries.indexOf(COLUMN, start);
|
|
175
|
+
return this.#entries.slice(start, column === -1 || column > end ? end : column);
|
|
176
|
+
}
|
|
117
177
|
/**
|
|
118
178
|
* How many words the dictionary holds, counting each script's key.
|
|
119
179
|
*/
|
|
120
180
|
get size() {
|
|
121
181
|
return this.#index.size;
|
|
122
182
|
}
|
|
183
|
+
/**
|
|
184
|
+
* The word at a position, or the empty string where there is none.
|
|
185
|
+
*
|
|
186
|
+
* Positions are what a second index over the dictionary stores, since a
|
|
187
|
+
* position is one number where a word is a string — the reverse index in
|
|
188
|
+
* `src/search/` is built on that. The same position indexes
|
|
189
|
+
* {@link Dictionary.frequencyAt} and {@link DictionaryReadings.readingAt}.
|
|
190
|
+
*/
|
|
191
|
+
wordAt(at) {
|
|
192
|
+
return this.#index.keyAt(at);
|
|
193
|
+
}
|
|
194
|
+
/**
|
|
195
|
+
* The frequency bucket of the word at a position, 0 rarest to 15 commonest.
|
|
196
|
+
*
|
|
197
|
+
* The bucket rather than {@link WordEntry.cost}, because ranking a candidate
|
|
198
|
+
* list wants the sixteen-value scale a counting sort runs on — and because
|
|
199
|
+
* reading it costs no decode, where asking for the entry would decode every
|
|
200
|
+
* candidate to learn one number. See {@link FrequencyTable.bucketOf}.
|
|
201
|
+
*/
|
|
202
|
+
frequencyAt(at) {
|
|
203
|
+
return this.#frequencies.bucketOf(at);
|
|
204
|
+
}
|
|
205
|
+
/**
|
|
206
|
+
* A cursor over every key's reading, in key order.
|
|
207
|
+
*
|
|
208
|
+
* Two passes, mirroring `readArtifact`: the character defaults are gathered
|
|
209
|
+
* first, because 83.25% of the full tier's keys store no reading and mean
|
|
210
|
+
* "the characters' defaults, in order". Holding the cursor holds those
|
|
211
|
+
* defaults, so build what needs it, use it, and let it go.
|
|
212
|
+
*/
|
|
213
|
+
readingsInOrder() {
|
|
214
|
+
const defaults = new Map();
|
|
215
|
+
for (let at = 0; at < this.#index.size; at++) {
|
|
216
|
+
const key = this.#index.keyAt(at);
|
|
217
|
+
const reading = isOneCharacter(key) ? this.#readingColumn(at) : "";
|
|
218
|
+
if (reading !== "") {
|
|
219
|
+
defaults.set(key, reading);
|
|
220
|
+
}
|
|
221
|
+
}
|
|
222
|
+
return {
|
|
223
|
+
size: this.#index.size,
|
|
224
|
+
readingAt: (at) => {
|
|
225
|
+
const stored = this.#readingColumn(at);
|
|
226
|
+
return stored === ""
|
|
227
|
+
? derivedReading(this.#index.keyAt(at), defaults)
|
|
228
|
+
: stored;
|
|
229
|
+
},
|
|
230
|
+
};
|
|
231
|
+
}
|
|
123
232
|
/**
|
|
124
233
|
* Whether any word begins with this text. An exact match counts.
|
|
125
234
|
*
|
|
@@ -156,13 +265,19 @@ export class Dictionary {
|
|
|
156
265
|
*
|
|
157
266
|
* The lattice needs these for its single-character fallback edges: 行 has to
|
|
158
267
|
* offer `xíng`, `háng` and `héng` at a position no word covers.
|
|
268
|
+
*
|
|
269
|
+
* The alternates are read off the canonical form's line, the same form
|
|
270
|
+
* {@link Dictionary.lookup} found the entry under. Searching the index for
|
|
271
|
+
* the raw character instead would land on the insertion point for a Hong Kong
|
|
272
|
+
* glyph form that is not itself a key — 裏 sits between whatever keys
|
|
273
|
+
* surround it — and take some unrelated entry's alternates.
|
|
159
274
|
*/
|
|
160
275
|
readingsOf(character) {
|
|
161
276
|
const entry = this.lookup(character);
|
|
162
277
|
if (entry === undefined) {
|
|
163
278
|
return [];
|
|
164
279
|
}
|
|
165
|
-
const found = this.#index.lookup(character);
|
|
280
|
+
const found = this.#index.lookup(toCanonicalGlyphs(character));
|
|
166
281
|
const alternates = (this.#columns(found.index)[4] ?? "")
|
|
167
282
|
.split(ALTERNATE)
|
|
168
283
|
.map((text) => decodeReading(text))
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"dictionary.js","sourceRoot":"","sources":["../../src/dictionary/dictionary.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,yBAAyB,CAAC;AACvD,OAAO,EAAE,iBAAiB,EAAE,MAAM,qBAAqB,CAAC;AAExD,OAAO,EAAE,aAAa,EAA2B,MAAM,eAAe,CAAC;AACvE,OAAO,EAAE,cAAc,EAAE,MAAM,sBAAsB,CAAC;AACtD,OAAO,EAAE,QAAQ,EAAE,MAAM,gBAAgB,CAAC;AAE1C;;GAEG;AACH,MAAM,MAAM,GAAG,IAAI,CAAC;AAEpB,MAAM,IAAI,GAAG,IAAI,CAAC;AAElB,MAAM,SAAS,GAAG,GAAG,CAAC;AAEtB,MAAM,gBAAgB,GAAG,GAAG,CAAC;AAE7B;;;;;;;GAOG;AACH,MAAM,eAAe,GAAG,0BAA0B,CAAC;
|
|
1
|
+
{"version":3,"file":"dictionary.js","sourceRoot":"","sources":["../../src/dictionary/dictionary.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,yBAAyB,CAAC;AACvD,OAAO,EAAE,iBAAiB,EAAE,MAAM,qBAAqB,CAAC;AAExD,OAAO,EAAE,aAAa,EAA2B,MAAM,eAAe,CAAC;AACvE,OAAO,EAAE,cAAc,EAAE,MAAM,sBAAsB,CAAC;AACtD,OAAO,EAAE,QAAQ,EAAE,MAAM,gBAAgB,CAAC;AAE1C;;GAEG;AACH,MAAM,MAAM,GAAG,IAAI,CAAC;AAEpB,MAAM,IAAI,GAAG,IAAI,CAAC;AAElB,MAAM,SAAS,GAAG,GAAG,CAAC;AAEtB,MAAM,gBAAgB,GAAG,GAAG,CAAC;AAE7B;;;;;;;GAOG;AACH,MAAM,eAAe,GAAG,0BAA0B,CAAC;AAEnD;;GAEG;AACH,MAAM,cAAc,GAAG,SAAS,CAAC;AAEjC;;;;;;GAMG;AACH,SAAS,cAAc,CAAC,GAAW;IACjC,IAAI,GAAG,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACrB,OAAO,IAAI,CAAC;IACd,CAAC;IACD,OAAO,GAAG,CAAC,MAAM,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,cAAc,CAAC;AACzE,CAAC;AAED;;;;;;;;;;;GAWG;AACH,SAAS,cAAc,CACrB,GAAW,EACX,QAAqC;IAErC,IAAI,MAAM,GAAG,EAAE,CAAC;IAChB,IAAI,EAAE,GAAG,CAAC,CAAC;IACX,OAAO,EAAE,GAAG,GAAG,CAAC,MAAM,EAAE,CAAC;QACvB,MAAM,KAAK,GAAG,CAAC,GAAG,CAAC,WAAW,CAAC,EAAE,CAAC,IAAI,CAAC,CAAC,IAAI,cAAc,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QACnE,MAAM,KAAK,GAAG,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,EAAE,EAAE,GAAG,KAAK,CAAC,CAAC,CAAC;QACtD,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;YACxB,OAAO,EAAE,CAAC;QACZ,CAAC;QACD,MAAM,GAAG,MAAM,KAAK,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,GAAG,MAAM,IAAI,KAAK,EAAE,CAAC;QACtD,EAAE,IAAI,KAAK,CAAC;IACd,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC;AAsDD;;;;;;;;;;;;;;;;;;GAkBG;AACH,MAAM,OAAO,UAAU;IACrB;;OAEG;IACH,MAAM,CAAC,IAAI,CAAC,QAA4B;QACtC,OAAO,IAAI,UAAU,CAAC,QAAQ,CAAC,CAAC;IAClC,CAAC;IAEQ,MAAM,CAAW;IACjB,QAAQ,CAAS;IACjB,WAAW,CAAc;IACzB,YAAY,CAAiB;IAC7B,QAAQ,GAAG,IAAI,GAAG,EAAqB,CAAC;IAEjD;;OAEG;IACH,YAAoB,QAA4B;QAC9C,IAAI,CAAC,MAAM,GAAG,QAAQ,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;QAC3C,IAAI,CAAC,QAAQ,GAAG,QAAQ,CAAC,OAAO,CAAC;QAEjC,MAAM,MAAM,GAAa,CAAC,CAAC,CAAC,CAAC;QAC7B,IAAI,SAAS,GAAG,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;QAC5C,OAAO,SAAS,KAAK,CAAC,CAAC,EAAE,CAAC;YACxB,MAAM,CAAC,IAAI,CAAC,SAAS,GAAG,CAAC,CAAC,CAAC;YAC3B,SAAS,GAAG,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,IAAI,EAAE,SAAS,GAAG,CAAC,CAAC,CAAC;QACzD,CAAC;QACD,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;QACtC,IAAI,CAAC,WAAW,GAAG,WAAW,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QAE5C,IAAI,CAAC,YAAY,GAAG,cAAc,CAAC,IAAI,CACrC,QAAQ,CAAC,WAAW,EACpB,IAAI,CAAC,MAAM,CAAC,IAAI,CACjB,CAAC;IACJ,CAAC;IAED;;OAEG;IACH,QAAQ,CAAC,EAAU;QACjB,MAAM,KAAK,GAAG,IAAI,CAAC,WAAW,CAAC,EAAE,CAAC,CAAC;QACnC,MAAM,SAAS,GAAG,IAAI,CAAC,WAAW,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC;QAC3C,2EAA2E;QAC3E,IAAI,KAAK,KAAK,SAAS,IAAI,SAAS,KAAK,SAAS,EAAE,CAAC;YACnD,OAAO,EAAE,CAAC;QACZ,CAAC;QACD,OAAO,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,KAAK,EAAE,SAAS,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;IACjE,CAAC;IAED;;;;;OAKG;IACH,cAAc,CAAC,IAAY;QACzB,MAAM,KAAK,GAAG,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;QACvC,OAAO,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;IACzE,CAAC;IAED;;OAEG;IACH,OAAO,CAAC,IAAY,EAAE,EAAU;QAC9B,MAAM,CAAC,OAAO,GAAG,EAAE,EAAE,MAAM,GAAG,EAAE,EAAE,YAAY,GAAG,EAAE,EAAE,KAAK,GAAG,EAAE,CAAC,GAC9D,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC;QAEpB,uEAAuE;QACvE,4DAA4D;QAC5D,MAAM,OAAO,GACX,OAAO,KAAK,EAAE;YACZ,CAAC,CAAC,YAAY,CAAC,IAAI,CAAC;iBACf,GAAG,CAAC,CAAC,SAAS,EAAE,EAAE,CAAC,IAAI,CAAC,cAAc,CAAC,SAAS,CAAC,IAAI,EAAE,CAAC;iBACxD,IAAI,CAAC,GAAG,CAAC;YACd,CAAC,CAAC,OAAO,CAAC;QACd,MAAM,OAAO,GAAG,aAAa,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC;QAC7C,MAAM,aAAa,GAAG,aAAa,CAAC,MAAM,CAAC,CAAC;QAC5C,MAAM,MAAM,GAAG,eAAe,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,CAAC,IAAI,CAAC,CAAC;QAC3D,MAAM,cAAc,GAClB,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;QAE5D,OAAO;YACL,IAAI;YACJ,OAAO,EAAE,OAAO;YAChB,GAAG,CAAC,aAAa,KAAK,SAAS,IAAI,EAAE,aAAa,EAAE,CAAC;YACrD,YAAY;YACZ,YAAY,EAAE,KAAK,CAAC,QAAQ,CAAC,gBAAgB,CAAC;YAC9C,GAAG,CAAC,cAAc,CAAC,MAAM,GAAG,CAAC,IAAI,EAAE,cAAc,EAAE,CAAC;YACpD,IAAI,EAAE,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC,EAAE,CAAC;SACnC,CAAC;IACJ,CAAC;IAED;;;;;;OAMG;IACH,cAAc,CAAC,EAAU;QACvB,MAAM,KAAK,GAAG,IAAI,CAAC,WAAW,CAAC,EAAE,CAAC,CAAC;QACnC,MAAM,SAAS,GAAG,IAAI,CAAC,WAAW,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC;QAC3C,IAAI,KAAK,KAAK,SAAS,IAAI,SAAS,KAAK,SAAS,EAAE,CAAC;YACnD,OAAO,EAAE,CAAC;QACZ,CAAC;QACD,MAAM,GAAG,GAAG,SAAS,GAAG,CAAC,CAAC;QAC1B,MAAM,MAAM,GAAG,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC;QACpD,OAAO,IAAI,CAAC,QAAQ,CAAC,KAAK,CACxB,KAAK,EACL,MAAM,KAAK,CAAC,CAAC,IAAI,MAAM,GAAG,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,CAC7C,CAAC;IACJ,CAAC;IAED;;OAEG;IACH,IAAI,IAAI;QACN,OAAO,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC;IAC1B,CAAC;IAED;;;;;;;OAOG;IACH,MAAM,CAAC,EAAU;QACf,OAAO,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;IAC/B,CAAC;IAED;;;;;;;OAOG;IACH,WAAW,CAAC,EAAU;QACpB,OAAO,IAAI,CAAC,YAAY,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC;IACxC,CAAC;IAED;;;;;;;OAOG;IACH,eAAe;QACb,MAAM,QAAQ,GAAG,IAAI,GAAG,EAAkB,CAAC;QAC3C,KAAK,IAAI,EAAE,GAAG,CAAC,EAAE,EAAE,GAAG,IAAI,CAAC,MAAM,CAAC,IAAI,EAAE,EAAE,EAAE,EAAE,CAAC;YAC7C,MAAM,GAAG,GAAG,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;YAClC,MAAM,OAAO,GAAG,cAAc,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,cAAc,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;YACnE,IAAI,OAAO,KAAK,EAAE,EAAE,CAAC;gBACnB,QAAQ,CAAC,GAAG,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;YAC7B,CAAC;QACH,CAAC;QAED,OAAO;YACL,IAAI,EAAE,IAAI,CAAC,MAAM,CAAC,IAAI;YACtB,SAAS,EAAE,CAAC,EAAU,EAAU,EAAE;gBAChC,MAAM,MAAM,GAAG,IAAI,CAAC,cAAc,CAAC,EAAE,CAAC,CAAC;gBACvC,OAAO,MAAM,KAAK,EAAE;oBAClB,CAAC,CAAC,cAAc,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,EAAE,QAAQ,CAAC;oBACjD,CAAC,CAAC,MAAM,CAAC;YACb,CAAC;SACF,CAAC;IACJ,CAAC;IAED;;;;;OAKG;IACH,SAAS,CAAC,IAAY;QACpB,OAAO,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,iBAAiB,CAAC,IAAI,CAAC,CAAC,CAAC;IACxD,CAAC;IAED;;;;;;;OAOG;IACH,MAAM,CAAC,IAAY;QACjB,MAAM,SAAS,GAAG,iBAAiB,CAAC,IAAI,CAAC,CAAC;QAC1C,MAAM,KAAK,GAAG,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;QAC5C,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC;YACjB,OAAO,SAAS,CAAC;QACnB,CAAC;QACD,MAAM,IAAI,GAAG,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QAC5C,IAAI,IAAI,KAAK,SAAS,EAAE,CAAC;YACvB,OAAO,IAAI,CAAC;QACd,CAAC;QACD,MAAM,KAAK,GAAG,IAAI,CAAC,OAAO,CAAC,SAAS,EAAE,KAAK,CAAC,KAAK,CAAC,CAAC;QACnD,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,KAAK,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC;QACtC,OAAO,KAAK,CAAC;IACf,CAAC;IAED;;;;;;;;;;;OAWG;IACH,UAAU,CAAC,SAAiB;QAC1B,MAAM,KAAK,GAAG,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;QACrC,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;YACxB,OAAO,EAAE,CAAC;QACZ,CAAC;QACD,MAAM,KAAK,GAAG,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,iBAAiB,CAAC,SAAS,CAAC,CAAC,CAAC;QAC/D,MAAM,UAAU,GAAG,CAAC,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;aACrD,KAAK,CAAC,SAAS,CAAC;aAChB,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,aAAa,CAAC,IAAI,CAAC,CAAC;aAClC,MAAM,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,KAAK,SAAS,CAAC,CAAC;QAC9C,OAAO,CAAC,KAAK,CAAC,OAAO,EAAE,GAAG,UAAU,CAAC,CAAC;IACxC,CAAC;CACF"}
|
|
@@ -16,6 +16,10 @@ export declare const NON_ERHUA_ER_WORDS: ReadonlySet<string>;
|
|
|
16
16
|
* Whether a word's last character is 儿, in either script.
|
|
17
17
|
*/
|
|
18
18
|
export declare function isErFinal(word: string): boolean;
|
|
19
|
+
/**
|
|
20
|
+
* Whether a character is the 儿 that 儿化 folds into the syllable before it.
|
|
21
|
+
*/
|
|
22
|
+
export declare function isErCharacter(character: string): boolean;
|
|
19
23
|
/**
|
|
20
24
|
* Add the r suffix of 儿化 to a syllable.
|
|
21
25
|
*
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"erhua.d.ts","sourceRoot":"","sources":["../../src/dictionary/erhua.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,yBAAyB,CAAC;AAYxD;;;;;;;;;;;GAWG;AACH,eAAO,MAAM,kBAAkB,EAAE,WAAW,CAAC,MAAM,CAiCjD,CAAC;AAEH;;GAEG;AACH,wBAAgB,SAAS,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAE/C;AAYD;;;;;;;;;;GAUG;AACH,wBAAgB,SAAS,CAAC,QAAQ,EAAE,QAAQ,GAAG,QAAQ,CAQtD;AAED;;;;;;;GAOG;AACH,wBAAgB,WAAW,CACzB,SAAS,EAAE,SAAS,QAAQ,EAAE,GAC7B,SAAS,QAAQ,EAAE,CAarB"}
|
|
1
|
+
{"version":3,"file":"erhua.d.ts","sourceRoot":"","sources":["../../src/dictionary/erhua.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,yBAAyB,CAAC;AAYxD;;;;;;;;;;;GAWG;AACH,eAAO,MAAM,kBAAkB,EAAE,WAAW,CAAC,MAAM,CAiCjD,CAAC;AAEH;;GAEG;AACH,wBAAgB,SAAS,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAE/C;AAED;;GAEG;AACH,wBAAgB,aAAa,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,CAExD;AAYD;;;;;;;;;;GAUG;AACH,wBAAgB,SAAS,CAAC,QAAQ,EAAE,QAAQ,GAAG,QAAQ,CAQtD;AAED;;;;;;;GAOG;AACH,wBAAgB,WAAW,CACzB,SAAS,EAAE,SAAS,QAAQ,EAAE,GAC7B,SAAS,QAAQ,EAAE,CAarB"}
|
package/dist/dictionary/erhua.js
CHANGED
|
@@ -58,6 +58,12 @@ export const NON_ERHUA_ER_WORDS = new Set([
|
|
|
58
58
|
export function isErFinal(word) {
|
|
59
59
|
return word.length > 1 && (word.endsWith(ER) || word.endsWith(ER_HANT));
|
|
60
60
|
}
|
|
61
|
+
/**
|
|
62
|
+
* Whether a character is the 儿 that 儿化 folds into the syllable before it.
|
|
63
|
+
*/
|
|
64
|
+
export function isErCharacter(character) {
|
|
65
|
+
return character === ER || character === ER_HANT;
|
|
66
|
+
}
|
|
61
67
|
/**
|
|
62
68
|
* Whether a syllable is a bare `er`, which is how the phrase corpus writes 儿化.
|
|
63
69
|
*
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"erhua.js","sourceRoot":"","sources":["../../src/dictionary/erhua.ts"],"names":[],"mappings":"AAEA;;GAEG;AACH,MAAM,EAAE,GAAG,GAAG,CAAC;AAEf;;GAEG;AACH,MAAM,OAAO,GAAG,GAAG,CAAC;AAEpB;;;;;;;;;;;GAWG;AACH,MAAM,CAAC,MAAM,kBAAkB,GAAwB,IAAI,GAAG,CAAC;IAC7D,GAAG;IACH,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,KAAK;IACL,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,KAAK;IACL,KAAK;IACL,KAAK;IACL,KAAK;IACL,KAAK;IACL,KAAK;IACL,MAAM;CACP,CAAC,CAAC;AAEH;;GAEG;AACH,MAAM,UAAU,SAAS,CAAC,IAAY;IACpC,OAAO,IAAI,CAAC,MAAM,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC,IAAI,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC;AAC1E,CAAC;AAED;;;;;GAKG;AACH,SAAS,QAAQ,CAAC,QAAkB;IAClC,OAAO,QAAQ,CAAC,OAAO,KAAK,EAAE,IAAI,QAAQ,CAAC,KAAK,KAAK,IAAI,CAAC;AAC5D,CAAC;AAED;;;;;;;;;;GAUG;AACH,MAAM,UAAU,SAAS,CAAC,QAAkB;IAC1C,IACE,QAAQ,CAAC,OAAO,KAAK,EAAE;QACvB,CAAC,QAAQ,CAAC,KAAK,KAAK,GAAG,IAAI,QAAQ,CAAC,KAAK,KAAK,IAAI,CAAC,EACnD,CAAC;QACD,OAAO,EAAE,OAAO,EAAE,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,IAAI,EAAE,QAAQ,CAAC,IAAI,EAAE,CAAC;IAC3D,CAAC;IACD,OAAO,EAAE,GAAG,QAAQ,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC;AACtC,CAAC;AAED;;;;;;;GAOG;AACH,MAAM,UAAU,WAAW,CACzB,SAA8B;IAE9B,MAAM,IAAI,GAAG,SAAS,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;IAC9B,MAAM,QAAQ,GAAG,SAAS,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;IAClC,IACE,SAAS,CAAC,MAAM,GAAG,CAAC;QACpB,IAAI,KAAK,SAAS;QAClB,QAAQ,KAAK,SAAS;QACtB,CAAC,QAAQ,CAAC,IAAI,CAAC;QACf,QAAQ,CAAC,KAAK,KAAK,IAAI,EACvB,CAAC;QACD,OAAO,SAAS,CAAC;IACnB,CAAC;IACD,OAAO,CAAC,GAAG,SAAS,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,SAAS,CAAC,QAAQ,CAAC,CAAC,CAAC;AAC1D,CAAC"}
|
|
1
|
+
{"version":3,"file":"erhua.js","sourceRoot":"","sources":["../../src/dictionary/erhua.ts"],"names":[],"mappings":"AAEA;;GAEG;AACH,MAAM,EAAE,GAAG,GAAG,CAAC;AAEf;;GAEG;AACH,MAAM,OAAO,GAAG,GAAG,CAAC;AAEpB;;;;;;;;;;;GAWG;AACH,MAAM,CAAC,MAAM,kBAAkB,GAAwB,IAAI,GAAG,CAAC;IAC7D,GAAG;IACH,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,KAAK;IACL,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,KAAK;IACL,KAAK;IACL,KAAK;IACL,KAAK;IACL,KAAK;IACL,KAAK;IACL,MAAM;CACP,CAAC,CAAC;AAEH;;GAEG;AACH,MAAM,UAAU,SAAS,CAAC,IAAY;IACpC,OAAO,IAAI,CAAC,MAAM,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC,IAAI,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC;AAC1E,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,aAAa,CAAC,SAAiB;IAC7C,OAAO,SAAS,KAAK,EAAE,IAAI,SAAS,KAAK,OAAO,CAAC;AACnD,CAAC;AAED;;;;;GAKG;AACH,SAAS,QAAQ,CAAC,QAAkB;IAClC,OAAO,QAAQ,CAAC,OAAO,KAAK,EAAE,IAAI,QAAQ,CAAC,KAAK,KAAK,IAAI,CAAC;AAC5D,CAAC;AAED;;;;;;;;;;GAUG;AACH,MAAM,UAAU,SAAS,CAAC,QAAkB;IAC1C,IACE,QAAQ,CAAC,OAAO,KAAK,EAAE;QACvB,CAAC,QAAQ,CAAC,KAAK,KAAK,GAAG,IAAI,QAAQ,CAAC,KAAK,KAAK,IAAI,CAAC,EACnD,CAAC;QACD,OAAO,EAAE,OAAO,EAAE,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,IAAI,EAAE,QAAQ,CAAC,IAAI,EAAE,CAAC;IAC3D,CAAC;IACD,OAAO,EAAE,GAAG,QAAQ,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC;AACtC,CAAC;AAED;;;;;;;GAOG;AACH,MAAM,UAAU,WAAW,CACzB,SAA8B;IAE9B,MAAM,IAAI,GAAG,SAAS,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;IAC9B,MAAM,QAAQ,GAAG,SAAS,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;IAClC,IACE,SAAS,CAAC,MAAM,GAAG,CAAC;QACpB,IAAI,KAAK,SAAS;QAClB,QAAQ,KAAK,SAAS;QACtB,CAAC,QAAQ,CAAC,IAAI,CAAC;QACf,QAAQ,CAAC,KAAK,KAAK,IAAI,EACvB,CAAC;QACD,OAAO,SAAS,CAAC;IACnB,CAAC;IACD,OAAO,CAAC,GAAG,SAAS,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,SAAS,CAAC,QAAQ,CAAC,CAAC,CAAC;AAC1D,CAAC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"merge.d.ts","sourceRoot":"","sources":["../../src/dictionary/merge.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,KAAK,WAAW,EAAoB,MAAM,sBAAsB,CAAC;AAC1E,OAAO,EAAmB,KAAK,UAAU,EAAE,MAAM,qBAAqB,CAAC;AACvE,OAAO,EAEL,KAAK,cAAc,EACnB,KAAK,cAAc,EACpB,MAAM,sBAAsB,CAAC;
|
|
1
|
+
{"version":3,"file":"merge.d.ts","sourceRoot":"","sources":["../../src/dictionary/merge.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,KAAK,WAAW,EAAoB,MAAM,sBAAsB,CAAC;AAC1E,OAAO,EAAmB,KAAK,UAAU,EAAE,MAAM,qBAAqB,CAAC;AACvE,OAAO,EAEL,KAAK,cAAc,EACnB,KAAK,cAAc,EACpB,MAAM,sBAAsB,CAAC;AAI9B,OAAO,EAAE,KAAK,eAAe,EAAiB,MAAM,YAAY,CAAC;AAejE;;GAEG;AACH,MAAM,WAAW,YAAY;IAC3B,QAAQ,CAAC,cAAc,EAAE,WAAW,CAAC,MAAM,EAAE,cAAc,CAAC,CAAC;IAC7D,QAAQ,CAAC,cAAc,EAAE,cAAc,CAAC;IACxC,QAAQ,CAAC,MAAM,EAAE,WAAW,CAAC,MAAM,EAAE,SAAS,MAAM,EAAE,CAAC,CAAC;IACxD,QAAQ,CAAC,MAAM,EAAE,SAAS,WAAW,EAAE,CAAC;IACxC,QAAQ,CAAC,KAAK,EAAE,WAAW,CAAC,MAAM,EAAE,UAAU,CAAC,CAAC;CACjD;AAED;;GAEG;AACH,MAAM,WAAW,UAAU;IACzB,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,iEAAiE;IACjE,QAAQ,CAAC,sBAAsB,EAAE,MAAM,CAAC;IACxC,qEAAqE;IACrE,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;IAC9B,wEAAwE;IACxE,QAAQ,CAAC,kBAAkB,EAAE,MAAM,CAAC;IACpC,0DAA0D;IAC1D,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,4DAA4D;IAC5D,QAAQ,CAAC,gBAAgB,EAAE,MAAM,CAAC;IAClC,8DAA8D;IAC9D,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAC;IAChC,sEAAsE;IACtE,QAAQ,CAAC,sBAAsB,EAAE,MAAM,CAAC;IACxC;;OAEG;IACH,QAAQ,CAAC,gBAAgB,EAAE,MAAM,CAAC;IAClC,8DAA8D;IAC9D,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAC;IAChC,6DAA6D;IAC7D,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;CAC3B;AAED;;GAEG;AACH,MAAM,WAAW,WAAW;IAC1B,yCAAyC;IACzC,QAAQ,CAAC,OAAO,EAAE,SAAS,eAAe,EAAE,CAAC;IAC7C,mEAAmE;IACnE,QAAQ,CAAC,QAAQ,EAAE,WAAW,CAAC,MAAM,EAAE,SAAS,MAAM,EAAE,CAAC,CAAC;IAC1D,QAAQ,CAAC,KAAK,EAAE,UAAU,CAAC;CAC5B;AA2WD;;;;;;;;GAQG;AACH,wBAAgB,YAAY,CAAC,OAAO,EAAE,YAAY,GAAG,WAAW,CA8V/D"}
|
package/dist/dictionary/merge.js
CHANGED
|
@@ -4,6 +4,7 @@ import { nameBoundariesOf } from "../sources/cedict.js";
|
|
|
4
4
|
import { isProperNounTag } from "../sources/jieba.js";
|
|
5
5
|
import { FREQUENCY_FIELD, } from "../sources/unihan.js";
|
|
6
6
|
import { NEUTRAL_TONE } from "../tone/tone.js";
|
|
7
|
+
import { countCorpusMass, rankByCorpusMass } from "./corpus-mass.js";
|
|
7
8
|
import { isSameReading } from "./entry.js";
|
|
8
9
|
import { attachErhua, isErFinal, NON_ERHUA_ER_WORDS } from "./erhua.js";
|
|
9
10
|
import { composeLocaleDeltas } from "./locale.js";
|
|
@@ -331,23 +332,30 @@ export function mergeSources(sources) {
|
|
|
331
332
|
// 誰 is `shéi` under 谁 — so the 简体 index alone cannot say what a character
|
|
332
333
|
// like that already reads in 普通话.
|
|
333
334
|
const cedictByHant = indexCedict(cedict, (entry) => entry.traditional);
|
|
335
|
+
// ── 繁體 evidence, mined from CC-CEDICT's own pairings ──────
|
|
336
|
+
// Before the character defaults, because the corpus mass those are ranked by
|
|
337
|
+
// has to reach 繁體 characters through this table: jieba's corpus and the
|
|
338
|
+
// phrase corpus are both 简体, so nothing else would count a vote for 髮.
|
|
339
|
+
const pairings = [];
|
|
340
|
+
for (const entry of cedict) {
|
|
341
|
+
const aligned = readAlignedReading(entry.simplified, entry.readings);
|
|
342
|
+
pairings.push(...pairScripts(entry.simplified, entry.traditional, aligned));
|
|
343
|
+
}
|
|
344
|
+
const traditional = TraditionalTable.build(pairings, unihanVariants, unihanReadings);
|
|
334
345
|
// ── Character defaults, which every later step leans on ────
|
|
346
|
+
// Unihan ranks the readings and the corpus re-ranks the ones it has seen: a
|
|
347
|
+
// character's default is whichever of its readings the dictionary's own words
|
|
348
|
+
// spend the most of jieba's corpus on, and Unihan's order decides the rest.
|
|
349
|
+
const mass = countCorpusMass({ phrase, cedict, jieba }, traditional);
|
|
335
350
|
const defaults = new Map();
|
|
336
351
|
for (const [character, readings] of unihanReadings) {
|
|
337
352
|
const parsed = resolveFrequencyTones(readings)
|
|
338
353
|
.map((reading) => characterSyllable(character, reading))
|
|
339
354
|
.filter((syllable) => syllable !== undefined);
|
|
340
355
|
if (parsed.length > 0) {
|
|
341
|
-
defaults.set(character, parsed);
|
|
356
|
+
defaults.set(character, rankByCorpusMass(character, parsed, readings.fields, mass));
|
|
342
357
|
}
|
|
343
358
|
}
|
|
344
|
-
// ── 繁體 evidence, mined from CC-CEDICT's own pairings ──────
|
|
345
|
-
const pairings = [];
|
|
346
|
-
for (const entry of cedict) {
|
|
347
|
-
const aligned = readAlignedReading(entry.simplified, entry.readings);
|
|
348
|
-
pairings.push(...pairScripts(entry.simplified, entry.traditional, aligned));
|
|
349
|
-
}
|
|
350
|
-
const traditional = TraditionalTable.build(pairings, unihanVariants, unihanReadings);
|
|
351
359
|
const words = new Set([
|
|
352
360
|
...defaults.keys(),
|
|
353
361
|
...[...phrase.keys()].filter((word) => !isSpeltTraditionally(word, cedictByWord, cedictByHant)),
|