@kensio/pinyinjs 1.7.8 → 1.7.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +18 -2
- package/data/core.entries +2259 -2259
- package/data/core.freq +0 -0
- package/data/full.counts +0 -0
- package/data/full.entries +2289 -2289
- package/data/full.freq +0 -0
- package/data/manifest.json +8 -8
- package/data/standard.entries +2257 -2257
- package/data/standard.freq +0 -0
- package/dist/cli/flag-lists.d.ts +2 -1
- package/dist/cli/flag-lists.d.ts.map +1 -1
- package/dist/cli/flag-lists.js.map +1 -1
- package/dist/cli/format-options.d.ts +5 -1
- package/dist/cli/format-options.d.ts.map +1 -1
- package/dist/cli/format-options.js +9 -2
- package/dist/cli/format-options.js.map +1 -1
- package/dist/cli/html-commands.d.ts.map +1 -1
- package/dist/cli/html-commands.js +15 -3
- package/dist/cli/html-commands.js.map +1 -1
- package/dist/cli/systems.d.ts +15 -43
- package/dist/cli/systems.d.ts.map +1 -1
- package/dist/cli/systems.js +9 -12
- package/dist/cli/systems.js.map +1 -1
- package/dist/decode/guo-rule.d.ts +64 -0
- package/dist/decode/guo-rule.d.ts.map +1 -0
- package/dist/decode/guo-rule.js +138 -0
- package/dist/decode/guo-rule.js.map +1 -0
- package/dist/decode/reading-rules.d.ts +1 -0
- package/dist/decode/reading-rules.d.ts.map +1 -1
- package/dist/decode/reading-rules.js +3 -0
- package/dist/decode/reading-rules.js.map +1 -1
- package/dist/dictionary/artifact.js +2 -2
- package/dist/dictionary/artifact.js.map +1 -1
- package/dist/dictionary/frequency-table.d.ts +103 -0
- package/dist/dictionary/frequency-table.d.ts.map +1 -1
- package/dist/dictionary/frequency-table.js +96 -8
- package/dist/dictionary/frequency-table.js.map +1 -1
- package/dist/dictionary/merge-types.d.ts +12 -0
- package/dist/dictionary/merge-types.d.ts.map +1 -1
- package/dist/dictionary/merge-types.js.map +1 -1
- package/dist/dictionary/merge-word.d.ts.map +1 -1
- package/dist/dictionary/merge-word.js +2 -2
- package/dist/dictionary/merge-word.js.map +1 -1
- package/dist/dictionary/merge.d.ts.map +1 -1
- package/dist/dictionary/merge.js +11 -1
- package/dist/dictionary/merge.js.map +1 -1
- package/dist/dictionary/name-mass.d.ts +42 -0
- package/dist/dictionary/name-mass.d.ts.map +1 -0
- package/dist/dictionary/name-mass.js +66 -0
- package/dist/dictionary/name-mass.js.map +1 -0
- package/dist/dictionary/proper-noun.d.ts +8 -1
- package/dist/dictionary/proper-noun.d.ts.map +1 -1
- package/dist/dictionary/proper-noun.js +13 -3
- package/dist/dictionary/proper-noun.js.map +1 -1
- package/dist/dictionary/script-pairings.d.ts +12 -0
- package/dist/dictionary/script-pairings.d.ts.map +1 -1
- package/dist/dictionary/script-pairings.js +20 -0
- package/dist/dictionary/script-pairings.js.map +1 -1
- package/dist/dictionary/traditional-carry.d.ts +23 -0
- package/dist/dictionary/traditional-carry.d.ts.map +1 -0
- package/dist/dictionary/traditional-carry.js +201 -0
- package/dist/dictionary/traditional-carry.js.map +1 -0
- package/dist/format/annotate.d.ts.map +1 -1
- package/dist/format/annotate.js +3 -3
- package/dist/format/annotate.js.map +1 -1
- package/dist/format/html.d.ts +41 -6
- package/dist/format/html.d.ts.map +1 -1
- package/dist/format/html.js +34 -83
- package/dist/format/html.js.map +1 -1
- package/dist/format/language-tag.d.ts +57 -0
- package/dist/format/language-tag.d.ts.map +1 -0
- package/dist/format/language-tag.js +76 -0
- package/dist/format/language-tag.js.map +1 -0
- package/dist/format/syllable-span.d.ts +10 -0
- package/dist/format/syllable-span.d.ts.map +1 -0
- package/dist/format/syllable-span.js +87 -0
- package/dist/format/syllable-span.js.map +1 -0
- package/dist/format/transcribed.d.ts +36 -0
- package/dist/format/transcribed.d.ts.map +1 -0
- package/dist/format/transcribed.js +38 -0
- package/dist/format/transcribed.js.map +1 -0
- package/dist/format/transcription.d.ts.map +1 -1
- package/dist/format/transcription.js +7 -19
- package/dist/format/transcription.js.map +1 -1
- package/dist/index.d.ts +3 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +2 -1
- package/dist/index.js.map +1 -1
- package/dist/transcription/systems.d.ts +82 -0
- package/dist/transcription/systems.d.ts.map +1 -0
- package/dist/transcription/systems.js +64 -0
- package/dist/transcription/systems.js.map +1 -0
- package/package.json +2 -2
- package/dist/cli/system-table.d.ts +0 -12
- package/dist/cli/system-table.d.ts.map +0 -1
- package/dist/cli/system-table.js +0 -48
- package/dist/cli/system-table.js.map +0 -1
|
@@ -8,6 +8,106 @@
|
|
|
8
8
|
* against another.
|
|
9
9
|
*/
|
|
10
10
|
export declare const FREQUENCY_BUCKETS = 16;
|
|
11
|
+
/**
|
|
12
|
+
* What every word in a decoded path costs before its frequency is counted.
|
|
13
|
+
*
|
|
14
|
+
* A bucket is `log f` on a scale where the most frequent word in the corpus
|
|
15
|
+
* reaches {@link MAX_BUCKET}, so bucket *b* stands for `log f = b · L / 15`
|
|
16
|
+
* where `L = log(1 + f_max)`. The decoder wants `−log P = log N − log f`, which
|
|
17
|
+
* on that same scale is `15 · log N / L − b`, so the constant term is
|
|
18
|
+
* `15 · (log N / L − 1)` rather than the 1 that was here before. That
|
|
19
|
+
* arithmetic is {@link derivedWordCharge}, and the build holds this constant to
|
|
20
|
+
* it, since a constant in `src/` is the one thing a corpus refresh cannot
|
|
21
|
+
* bring with it.
|
|
22
|
+
*
|
|
23
|
+
* **`N` is jieba's corpus**, which is where every count in the dictionary comes
|
|
24
|
+
* from. As the build parses it, 60,101,964 occurrences over 349,045 words with
|
|
25
|
+
* 了 the busiest at 883,634, giving `log N = 17.91`, `L = 13.69` and a charge
|
|
26
|
+
* of 4.62. Only the *ratio* of the two logarithms matters, and it moves slowly.
|
|
27
|
+
* Doubling every count in the corpus shifts it by 0.05.
|
|
28
|
+
*
|
|
29
|
+
* **Summing `full.counts` gives 82,372,768, and that is not `N`.** The artifact
|
|
30
|
+
* attributes one corpus to more keys than jieba has words. A 繁體 spelling
|
|
31
|
+
* claims its 简体 entry's count, and `traditional-carry.ts` lends a count to a
|
|
32
|
+
* 繁體 character holding an entry of its own. 银行 and 銀行 are one word met
|
|
33
|
+
* once and two keys counted twice, and 20,447,271 of that total sits on keys
|
|
34
|
+
* jieba does not list at all.
|
|
35
|
+
*
|
|
36
|
+
* The value is load-bearing. At the old charge of 1 the lattice decoder read
|
|
37
|
+
* 还给 as 还 + 给 and 还是 as 还 + 是, because two common characters summed to
|
|
38
|
+
* less than one uncommon word. At the derived charge both stay whole.
|
|
39
|
+
*
|
|
40
|
+
* **Its exact value inside a unit is not.** Buckets are integers, so a path
|
|
41
|
+
* carrying *k* words more than its rival turns on where `k · charge` falls
|
|
42
|
+
* between two of them. 4.62 and 4.97 agree at `k` of 1 and 2 and first differ
|
|
43
|
+
* at 3. Converted at 4.97, 63 of the 139,682 Han runs in 88,866 lines of
|
|
44
|
+
* Tatoeba and zh.wikipedia come out differently, and every one of the 63
|
|
45
|
+
* decodes into as many words as it did before. They are exact ties in the
|
|
46
|
+
* bucket sum, re-broken by the order the floats happened to be added in.
|
|
47
|
+
*/
|
|
48
|
+
export declare const WORD_CHARGE = 4.62;
|
|
49
|
+
/**
|
|
50
|
+
* The count a name the corpus never counted is quantised at.
|
|
51
|
+
*
|
|
52
|
+
* A count of zero earns bucket 0, and {@link FrequencyTable.costOf} reads that
|
|
53
|
+
* as `−log P` with `P` of `1/N`. On jieba's corpus the flat cost it produces is
|
|
54
|
+
* 19.62, and two mid-frequency characters come to less. 脸书 cost 19.62 where
|
|
55
|
+
* 脸 and 书 together cost 18.24, and `decodeRun` read Facebook as a face and a
|
|
56
|
+
* book. 推特, 高雄, 乐高 and 网飞 all went the same way, and 671 of the 5,039
|
|
57
|
+
* uncounted names of two or three characters came apart when read on their
|
|
58
|
+
* own. 谷歌 survived on an exact tie.
|
|
59
|
+
*
|
|
60
|
+
* **Zero is the wrong count for a name jieba's list never held.** The list has
|
|
61
|
+
* no frequency floor to be under. It carries 一万一千五百二十颗 on two
|
|
62
|
+
* occurrences. A word missing from it was left out of the lexicon, and the
|
|
63
|
+
* corpus was never asked about it. Names are where that bites, since a name
|
|
64
|
+
* enters the language whenever something is given one. 微软 was counted 1757
|
|
65
|
+
* times and 脸书 not at all, and only one of the two existed when the list was
|
|
66
|
+
* drawn. 10,676 of the dictionary's 42,861 names are counted nowhere.
|
|
67
|
+
*
|
|
68
|
+
* **3 is jieba's own answer for a word it lists and cannot count.** Its counts
|
|
69
|
+
* bottom out at 2, and 3 carries 159,318 of 349,045 entries against 40,502 at 2
|
|
70
|
+
* and 12,679 at 4. Interpolating between the neighbours accounts for about
|
|
71
|
+
* 22,000 of those, and the other 137,000 carry a default. It is the default a
|
|
72
|
+
* hand-added entry gets (AT&T, B超 and C++ are all counted 3).
|
|
73
|
+
*
|
|
74
|
+
* The value is loose. Bucket 2 runs from 3 to 8, so any count jieba might have
|
|
75
|
+
* settled on in that range quantises the same way and lands the cost at 17.62.
|
|
76
|
+
* What the floor buys a name is two buckets against a split.
|
|
77
|
+
*
|
|
78
|
+
* **Names only.** The same floor over every uncounted key lifts two thirds of
|
|
79
|
+
* the dictionary by two buckets and re-tips
|
|
80
|
+
* {@link import("../decode/lattice-types.js").READING_CHARGE}, which was sized
|
|
81
|
+
* against bucket 0 at 19.62. Measured that way 從容地 joins, 都會 reads `dūhuì`
|
|
82
|
+
* and 過得 reads `guò dé`, and the gold harness goes from 3 misses to 6. The
|
|
83
|
+
* dictionary's tail is where 费耗, 块肉 and 拍张 live, and bucket 0 is what
|
|
84
|
+
* holds them off.
|
|
85
|
+
*
|
|
86
|
+
* 428 of the 671 names now stay whole. The ones left are compounds a split
|
|
87
|
+
* suits, 中国菜, 日本人, 上海话 and 美国人 among them, and each beats its own
|
|
88
|
+
* entry by more than two buckets. Over the 88,866 lines of Tatoeba and
|
|
89
|
+
* zh.wikipedia 86 of 139,682 Han runs move, the gold harness holds at 3 misses
|
|
90
|
+
* and CPP holds at 91.49%.
|
|
91
|
+
*/
|
|
92
|
+
export declare const UNCOUNTED_NAME = 3;
|
|
93
|
+
/**
|
|
94
|
+
* The count an entry is quantised at. Its own, unless it is a name the corpus
|
|
95
|
+
* never counted. See {@link UNCOUNTED_NAME}.
|
|
96
|
+
*
|
|
97
|
+
* Applied where the artifact is written and nowhere else, so `full.counts`
|
|
98
|
+
* keeps the corpus as it was met. The two files disagree by design over these
|
|
99
|
+
* 10,676 keys, and `data.test.ts` checks their alignment through this
|
|
100
|
+
* function.
|
|
101
|
+
*/
|
|
102
|
+
export declare function countForQuantising(frequency: number, isProperNoun: boolean): number;
|
|
103
|
+
/**
|
|
104
|
+
* The charge a corpus derives, for the build to hold {@link WORD_CHARGE} to.
|
|
105
|
+
*
|
|
106
|
+
* `total` is the corpus, counted once. `highest` is the largest count in it,
|
|
107
|
+
* which is what {@link FrequencyTable.build} scales the buckets by, so the two
|
|
108
|
+
* have to be read off the same corpus for the result to mean anything.
|
|
109
|
+
*/
|
|
110
|
+
export declare function derivedWordCharge(total: number, highest: number): number;
|
|
11
111
|
/**
|
|
12
112
|
* Quantised word frequencies, packed two to a byte.
|
|
13
113
|
*
|
|
@@ -25,6 +125,9 @@ export declare class FrequencyTable {
|
|
|
25
125
|
* the word was not attested, which still earns the lowest bucket rather than
|
|
26
126
|
* being excluded — an unattested word in the dictionary is rare, not
|
|
27
127
|
* impossible.
|
|
128
|
+
*
|
|
129
|
+
* Counts arrive through {@link countForQuantising}, which is where a name the
|
|
130
|
+
* corpus never counted is given jieba's own default for one.
|
|
28
131
|
*/
|
|
29
132
|
static build(frequencies: readonly number[]): FrequencyTable;
|
|
30
133
|
/**
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"frequency-table.d.ts","sourceRoot":"","sources":["../../src/dictionary/frequency-table.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AACH,eAAO,MAAM,iBAAiB,KAAK,CAAC;
|
|
1
|
+
{"version":3,"file":"frequency-table.d.ts","sourceRoot":"","sources":["../../src/dictionary/frequency-table.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AACH,eAAO,MAAM,iBAAiB,KAAK,CAAC;AAIpC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAoCG;AACH,eAAO,MAAM,WAAW,OAAO,CAAC;AAEhC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA0CG;AACH,eAAO,MAAM,cAAc,IAAI,CAAC;AAEhC;;;;;;;;GAQG;AACH,wBAAgB,kBAAkB,CAChC,SAAS,EAAE,MAAM,EACjB,YAAY,EAAE,OAAO,GACpB,MAAM,CAER;AAED;;;;;;GAMG;AACH,wBAAgB,iBAAiB,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,GAAG,MAAM,CAExE;AAED;;;;;;GAMG;AACH,qBAAa,cAAc;;IACzB;;;;;;;;;;;OAWG;IACH,MAAM,CAAC,KAAK,CAAC,WAAW,EAAE,SAAS,MAAM,EAAE,GAAG,cAAc,CA4B3D;IAED;;OAEG;IACH,MAAM,CAAC,IAAI,CAAC,MAAM,EAAE,UAAU,EAAE,IAAI,EAAE,MAAM,GAAG,cAAc,CAE5D;IAKD;;;OAGG;IACH,OAAO,eAGN;IAED;;OAEG;IACH,IAAI,IAAI,IAAI,MAAM,CAEjB;IAED;;;;;OAKG;IACH,QAAQ,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAM9B;IAED;;;;;;;;OAQG;IACH,MAAM,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAE5B;IAED;;OAEG;IACH,SAAS,IAAI,UAAU,CAEtB;CACF"}
|
|
@@ -15,19 +15,104 @@ const MAX_BUCKET = FREQUENCY_BUCKETS - 1;
|
|
|
15
15
|
* A bucket is `log f` on a scale where the most frequent word in the corpus
|
|
16
16
|
* reaches {@link MAX_BUCKET}, so bucket *b* stands for `log f = b · L / 15`
|
|
17
17
|
* where `L = log(1 + f_max)`. The decoder wants `−log P = log N − log f`, which
|
|
18
|
-
* on that same scale is `15 · log N / L − b
|
|
19
|
-
* `15 · (log N / L − 1)` rather than the 1 that was here before.
|
|
18
|
+
* on that same scale is `15 · log N / L − b`, so the constant term is
|
|
19
|
+
* `15 · (log N / L − 1)` rather than the 1 that was here before. That
|
|
20
|
+
* arithmetic is {@link derivedWordCharge}, and the build holds this constant to
|
|
21
|
+
* it, since a constant in `src/` is the one thing a corpus refresh cannot
|
|
22
|
+
* bring with it.
|
|
20
23
|
*
|
|
21
|
-
*
|
|
22
|
-
*
|
|
23
|
-
*
|
|
24
|
-
*
|
|
24
|
+
* **`N` is jieba's corpus**, which is where every count in the dictionary comes
|
|
25
|
+
* from. As the build parses it, 60,101,964 occurrences over 349,045 words with
|
|
26
|
+
* 了 the busiest at 883,634, giving `log N = 17.91`, `L = 13.69` and a charge
|
|
27
|
+
* of 4.62. Only the *ratio* of the two logarithms matters, and it moves slowly.
|
|
28
|
+
* Doubling every count in the corpus shifts it by 0.05.
|
|
29
|
+
*
|
|
30
|
+
* **Summing `full.counts` gives 82,372,768, and that is not `N`.** The artifact
|
|
31
|
+
* attributes one corpus to more keys than jieba has words. A 繁體 spelling
|
|
32
|
+
* claims its 简体 entry's count, and `traditional-carry.ts` lends a count to a
|
|
33
|
+
* 繁體 character holding an entry of its own. 银行 and 銀行 are one word met
|
|
34
|
+
* once and two keys counted twice, and 20,447,271 of that total sits on keys
|
|
35
|
+
* jieba does not list at all.
|
|
25
36
|
*
|
|
26
37
|
* The value is load-bearing. At the old charge of 1 the lattice decoder read
|
|
27
38
|
* 还给 as 还 + 给 and 还是 as 还 + 是, because two common characters summed to
|
|
28
|
-
* less than one uncommon word
|
|
39
|
+
* less than one uncommon word. At the derived charge both stay whole.
|
|
40
|
+
*
|
|
41
|
+
* **Its exact value inside a unit is not.** Buckets are integers, so a path
|
|
42
|
+
* carrying *k* words more than its rival turns on where `k · charge` falls
|
|
43
|
+
* between two of them. 4.62 and 4.97 agree at `k` of 1 and 2 and first differ
|
|
44
|
+
* at 3. Converted at 4.97, 63 of the 139,682 Han runs in 88,866 lines of
|
|
45
|
+
* Tatoeba and zh.wikipedia come out differently, and every one of the 63
|
|
46
|
+
* decodes into as many words as it did before. They are exact ties in the
|
|
47
|
+
* bucket sum, re-broken by the order the floats happened to be added in.
|
|
48
|
+
*/
|
|
49
|
+
export const WORD_CHARGE = 4.62;
|
|
50
|
+
/**
|
|
51
|
+
* The count a name the corpus never counted is quantised at.
|
|
52
|
+
*
|
|
53
|
+
* A count of zero earns bucket 0, and {@link FrequencyTable.costOf} reads that
|
|
54
|
+
* as `−log P` with `P` of `1/N`. On jieba's corpus the flat cost it produces is
|
|
55
|
+
* 19.62, and two mid-frequency characters come to less. 脸书 cost 19.62 where
|
|
56
|
+
* 脸 and 书 together cost 18.24, and `decodeRun` read Facebook as a face and a
|
|
57
|
+
* book. 推特, 高雄, 乐高 and 网飞 all went the same way, and 671 of the 5,039
|
|
58
|
+
* uncounted names of two or three characters came apart when read on their
|
|
59
|
+
* own. 谷歌 survived on an exact tie.
|
|
60
|
+
*
|
|
61
|
+
* **Zero is the wrong count for a name jieba's list never held.** The list has
|
|
62
|
+
* no frequency floor to be under. It carries 一万一千五百二十颗 on two
|
|
63
|
+
* occurrences. A word missing from it was left out of the lexicon, and the
|
|
64
|
+
* corpus was never asked about it. Names are where that bites, since a name
|
|
65
|
+
* enters the language whenever something is given one. 微软 was counted 1757
|
|
66
|
+
* times and 脸书 not at all, and only one of the two existed when the list was
|
|
67
|
+
* drawn. 10,676 of the dictionary's 42,861 names are counted nowhere.
|
|
68
|
+
*
|
|
69
|
+
* **3 is jieba's own answer for a word it lists and cannot count.** Its counts
|
|
70
|
+
* bottom out at 2, and 3 carries 159,318 of 349,045 entries against 40,502 at 2
|
|
71
|
+
* and 12,679 at 4. Interpolating between the neighbours accounts for about
|
|
72
|
+
* 22,000 of those, and the other 137,000 carry a default. It is the default a
|
|
73
|
+
* hand-added entry gets (AT&T, B超 and C++ are all counted 3).
|
|
74
|
+
*
|
|
75
|
+
* The value is loose. Bucket 2 runs from 3 to 8, so any count jieba might have
|
|
76
|
+
* settled on in that range quantises the same way and lands the cost at 17.62.
|
|
77
|
+
* What the floor buys a name is two buckets against a split.
|
|
78
|
+
*
|
|
79
|
+
* **Names only.** The same floor over every uncounted key lifts two thirds of
|
|
80
|
+
* the dictionary by two buckets and re-tips
|
|
81
|
+
* {@link import("../decode/lattice-types.js").READING_CHARGE}, which was sized
|
|
82
|
+
* against bucket 0 at 19.62. Measured that way 從容地 joins, 都會 reads `dūhuì`
|
|
83
|
+
* and 過得 reads `guò dé`, and the gold harness goes from 3 misses to 6. The
|
|
84
|
+
* dictionary's tail is where 费耗, 块肉 and 拍张 live, and bucket 0 is what
|
|
85
|
+
* holds them off.
|
|
86
|
+
*
|
|
87
|
+
* 428 of the 671 names now stay whole. The ones left are compounds a split
|
|
88
|
+
* suits, 中国菜, 日本人, 上海话 and 美国人 among them, and each beats its own
|
|
89
|
+
* entry by more than two buckets. Over the 88,866 lines of Tatoeba and
|
|
90
|
+
* zh.wikipedia 86 of 139,682 Han runs move, the gold harness holds at 3 misses
|
|
91
|
+
* and CPP holds at 91.49%.
|
|
29
92
|
*/
|
|
30
|
-
const
|
|
93
|
+
export const UNCOUNTED_NAME = 3;
|
|
94
|
+
/**
|
|
95
|
+
* The count an entry is quantised at. Its own, unless it is a name the corpus
|
|
96
|
+
* never counted. See {@link UNCOUNTED_NAME}.
|
|
97
|
+
*
|
|
98
|
+
* Applied where the artifact is written and nowhere else, so `full.counts`
|
|
99
|
+
* keeps the corpus as it was met. The two files disagree by design over these
|
|
100
|
+
* 10,676 keys, and `data.test.ts` checks their alignment through this
|
|
101
|
+
* function.
|
|
102
|
+
*/
|
|
103
|
+
export function countForQuantising(frequency, isProperNoun) {
|
|
104
|
+
return isProperNoun && frequency <= 0 ? UNCOUNTED_NAME : frequency;
|
|
105
|
+
}
|
|
106
|
+
/**
|
|
107
|
+
* The charge a corpus derives, for the build to hold {@link WORD_CHARGE} to.
|
|
108
|
+
*
|
|
109
|
+
* `total` is the corpus, counted once. `highest` is the largest count in it,
|
|
110
|
+
* which is what {@link FrequencyTable.build} scales the buckets by, so the two
|
|
111
|
+
* have to be read off the same corpus for the result to mean anything.
|
|
112
|
+
*/
|
|
113
|
+
export function derivedWordCharge(total, highest) {
|
|
114
|
+
return MAX_BUCKET * (Math.log(total) / Math.log1p(highest) - 1);
|
|
115
|
+
}
|
|
31
116
|
/**
|
|
32
117
|
* Quantised word frequencies, packed two to a byte.
|
|
33
118
|
*
|
|
@@ -44,6 +129,9 @@ export class FrequencyTable {
|
|
|
44
129
|
* the word was not attested, which still earns the lowest bucket rather than
|
|
45
130
|
* being excluded — an unattested word in the dictionary is rare, not
|
|
46
131
|
* impossible.
|
|
132
|
+
*
|
|
133
|
+
* Counts arrive through {@link countForQuantising}, which is where a name the
|
|
134
|
+
* corpus never counted is given jieba's own default for one.
|
|
47
135
|
*/
|
|
48
136
|
static build(frequencies) {
|
|
49
137
|
// Looped rather than `Math.max(0, ...frequencies)`, which passes one
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"frequency-table.js","sourceRoot":"","sources":["../../src/dictionary/frequency-table.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AACH,MAAM,CAAC,MAAM,iBAAiB,GAAG,EAAE,CAAC;AAEpC,MAAM,UAAU,GAAG,iBAAiB,GAAG,CAAC,CAAC;AAEzC
|
|
1
|
+
{"version":3,"file":"frequency-table.js","sourceRoot":"","sources":["../../src/dictionary/frequency-table.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AACH,MAAM,CAAC,MAAM,iBAAiB,GAAG,EAAE,CAAC;AAEpC,MAAM,UAAU,GAAG,iBAAiB,GAAG,CAAC,CAAC;AAEzC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAoCG;AACH,MAAM,CAAC,MAAM,WAAW,GAAG,IAAI,CAAC;AAEhC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA0CG;AACH,MAAM,CAAC,MAAM,cAAc,GAAG,CAAC,CAAC;AAEhC;;;;;;;;GAQG;AACH,MAAM,UAAU,kBAAkB,CAChC,SAAiB,EACjB,YAAqB;IAErB,OAAO,YAAY,IAAI,SAAS,IAAI,CAAC,CAAC,CAAC,CAAC,cAAc,CAAC,CAAC,CAAC,SAAS,CAAC;AACrE,CAAC;AAED;;;;;;GAMG;AACH,MAAM,UAAU,iBAAiB,CAAC,KAAa,EAAE,OAAe;IAC9D,OAAO,UAAU,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC;AAClE,CAAC;AAED;;;;;;GAMG;AACH,MAAM,OAAO,cAAc;IACzB;;;;;;;;;;;OAWG;IACH,MAAM,CAAC,KAAK,CAAC,WAA8B;QACzC,qEAAqE;QACrE,4EAA4E;QAC5E,0EAA0E;QAC1E,IAAI,OAAO,GAAG,CAAC,CAAC;QAChB,KAAK,MAAM,SAAS,IAAI,WAAW,EAAE,CAAC;YACpC,IAAI,SAAS,GAAG,OAAO,EAAE,CAAC;gBACxB,OAAO,GAAG,SAAS,CAAC;YACtB,CAAC;QACH,CAAC;QACD,MAAM,KAAK,GAAG,OAAO,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,UAAU,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;QAEnE,MAAM,MAAM,GAAG,IAAI,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,WAAW,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC;QACjE,KAAK,MAAM,CAAC,KAAK,EAAE,SAAS,CAAC,IAAI,WAAW,CAAC,OAAO,EAAE,EAAE,CAAC;YACvD,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,SAAS,CAAC,CAAC;YACxC,MAAM,MAAM,GAAG,IAAI,CAAC,GAAG,CACrB,UAAU,EACV,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,GAAG,KAAK,CAAC,CACzC,CAAC;YACF,MAAM,SAAS,GAAG,KAAK,IAAI,CAAC,CAAC;YAC7B,MAAM,QAAQ,GAAG,MAAM,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;YACxC,MAAM,CAAC,SAAS,CAAC;gBACf,KAAK,GAAG,CAAC,KAAK,CAAC;oBACb,CAAC,CAAC,CAAC,MAAM,IAAI,CAAC,CAAC,GAAG,CAAC,QAAQ,GAAG,GAAG,CAAC;oBAClC,CAAC,CAAC,CAAC,QAAQ,GAAG,IAAI,CAAC,GAAG,MAAM,CAAC;QACnC,CAAC;QAED,OAAO,IAAI,cAAc,CAAC,MAAM,EAAE,WAAW,CAAC,MAAM,CAAC,CAAC;IACxD,CAAC;IAED;;OAEG;IACH,MAAM,CAAC,IAAI,CAAC,MAAkB,EAAE,IAAY;QAC1C,OAAO,IAAI,cAAc,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;IAC1C,CAAC;IAEQ,OAAO,CAAa;IACpB,KAAK,CAAS;IAEvB;;;OAGG;IACH,YAAoB,MAAkB,EAAE,IAAY;QAClD,IAAI,CAAC,OAAO,GAAG,MAAM,CAAC;QACtB,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC;IACpB,CAAC;IAED;;OAEG;IACH,IAAI,IAAI;QACN,OAAO,IAAI,CAAC,KAAK,CAAC;IACpB,CAAC;IAED;;;;;OAKG;IACH,QAAQ,CAAC,KAAa;QACpB,IAAI,KAAK,GAAG,CAAC,IAAI,KAAK,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC;YACrC,OAAO,CAAC,CAAC;QACX,CAAC;QACD,MAAM,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,KAAK,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC;QAC3C,OAAO,KAAK,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,GAAG,GAAG,CAAC;IAClD,CAAC;IAED;;;;;;;;OAQG;IACH,MAAM,CAAC,KAAa;QAClB,OAAO,UAAU,GAAG,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,GAAG,WAAW,CAAC;IACzD,CAAC;IAED;;OAEG;IACH,SAAS;QACP,OAAO,IAAI,CAAC,OAAO,CAAC;IACtB,CAAC;CACF"}
|
|
@@ -9,6 +9,7 @@ import type { JiebaEntry } from "../sources/jieba.js";
|
|
|
9
9
|
import type { UnihanReadings, UnihanVariants } from "../sources/unihan.js";
|
|
10
10
|
import type { Syllable } from "../syllable/syllable.js";
|
|
11
11
|
import type { DictionaryEntry } from "./entry.js";
|
|
12
|
+
import type { NameMassTable } from "./name-mass.js";
|
|
12
13
|
import type { TraditionalTable } from "./traditional-table.js";
|
|
13
14
|
/**
|
|
14
15
|
* The parsed sources the merge combines.
|
|
@@ -62,6 +63,15 @@ export interface MergeStats {
|
|
|
62
63
|
readonly properNounVetoes: number;
|
|
63
64
|
/** Entries whose parts CC-CEDICT's capitalisation divides. */
|
|
64
65
|
readonly nameBoundaries: number;
|
|
66
|
+
/**
|
|
67
|
+
* 繁體 spellings given the part of speech their 简体 word carries. See
|
|
68
|
+
* {@link import("./traditional-carry.js").carryToTraditional}.
|
|
69
|
+
*/
|
|
70
|
+
readonly carriedTags: number;
|
|
71
|
+
/** 繁體 spellings given the corpus count their 简体 word carries. */
|
|
72
|
+
readonly carriedCounts: number;
|
|
73
|
+
/** 繁體 spellings given the proper-noun bit their 简体 word carries. */
|
|
74
|
+
readonly carriedCapitals: number;
|
|
65
75
|
/** Words dropped because no source gave a usable reading. */
|
|
66
76
|
readonly rejected: number;
|
|
67
77
|
}
|
|
@@ -87,6 +97,8 @@ export interface WordSources {
|
|
|
87
97
|
readonly cedictByHant: ReadonlyMap<string, readonly CedictEntry[]>;
|
|
88
98
|
readonly phrase: ReadonlyMap<string, readonly string[]>;
|
|
89
99
|
readonly jieba: ReadonlyMap<string, JiebaEntry>;
|
|
100
|
+
/** What each character heads in jieba's word list, for {@link import("./proper-noun.js").properNounOf}. */
|
|
101
|
+
readonly nameMass: NameMassTable;
|
|
90
102
|
readonly unihanReadings: ReadonlyMap<string, UnihanReadings>;
|
|
91
103
|
readonly traditional: TraditionalTable;
|
|
92
104
|
readonly defaults: ReadonlyMap<string, readonly Syllable[]>;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"merge-types.d.ts","sourceRoot":"","sources":["../../src/dictionary/merge-types.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AACH,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,sBAAsB,CAAC;AACxD,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,qBAAqB,CAAC;AACtD,OAAO,KAAK,EAAE,cAAc,EAAE,cAAc,EAAE,MAAM,sBAAsB,CAAC;AAC3E,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,yBAAyB,CAAC;AACxD,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,YAAY,CAAC;AAClD,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,wBAAwB,CAAC;AAE/D;;GAEG;AACH,MAAM,WAAW,YAAY;IAC3B,QAAQ,CAAC,cAAc,EAAE,WAAW,CAAC,MAAM,EAAE,cAAc,CAAC,CAAC;IAC7D,QAAQ,CAAC,cAAc,EAAE,cAAc,CAAC;IACxC,QAAQ,CAAC,MAAM,EAAE,WAAW,CAAC,MAAM,EAAE,SAAS,MAAM,EAAE,CAAC,CAAC;IACxD,QAAQ,CAAC,MAAM,EAAE,SAAS,WAAW,EAAE,CAAC;IACxC,QAAQ,CAAC,KAAK,EAAE,WAAW,CAAC,MAAM,EAAE,UAAU,CAAC,CAAC;CACjD;AAED;;GAEG;AACH,MAAM,WAAW,UAAU;IACzB,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,iEAAiE;IACjE,QAAQ,CAAC,sBAAsB,EAAE,MAAM,CAAC;IACxC;;;OAGG;IACH,QAAQ,CAAC,eAAe,EAAE,MAAM,CAAC;IACjC,qEAAqE;IACrE,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;IAC9B;;;OAGG;IACH,QAAQ,CAAC,mBAAmB,EAAE,MAAM,CAAC;IACrC,wEAAwE;IACxE,QAAQ,CAAC,kBAAkB,EAAE,MAAM,CAAC;IACpC,0DAA0D;IAC1D,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,4DAA4D;IAC5D,QAAQ,CAAC,gBAAgB,EAAE,MAAM,CAAC;IAClC;;;OAGG;IACH,QAAQ,CAAC,oBAAoB,EAAE,MAAM,CAAC;IACtC,8DAA8D;IAC9D,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAC;IAChC,sEAAsE;IACtE,QAAQ,CAAC,sBAAsB,EAAE,MAAM,CAAC;IACxC;;OAEG;IACH,QAAQ,CAAC,gBAAgB,EAAE,MAAM,CAAC;IAClC,8DAA8D;IAC9D,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAC;IAChC,6DAA6D;IAC7D,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;CAC3B;AAED;;GAEG;AACH,MAAM,WAAW,WAAW;IAC1B,yCAAyC;IACzC,QAAQ,CAAC,OAAO,EAAE,SAAS,eAAe,EAAE,CAAC;IAC7C,mEAAmE;IACnE,QAAQ,CAAC,QAAQ,EAAE,WAAW,CAAC,MAAM,EAAE,SAAS,MAAM,EAAE,CAAC,CAAC;IAC1D,QAAQ,CAAC,KAAK,EAAE,UAAU,CAAC;CAC5B;AAED;;;;;;GAMG;AACH,MAAM,WAAW,WAAW;IAC1B,QAAQ,CAAC,YAAY,EAAE,WAAW,CAAC,MAAM,EAAE,SAAS,WAAW,EAAE,CAAC,CAAC;IACnE,QAAQ,CAAC,YAAY,EAAE,WAAW,CAAC,MAAM,EAAE,SAAS,WAAW,EAAE,CAAC,CAAC;IACnE,QAAQ,CAAC,MAAM,EAAE,WAAW,CAAC,MAAM,EAAE,SAAS,MAAM,EAAE,CAAC,CAAC;IACxD,QAAQ,CAAC,KAAK,EAAE,WAAW,CAAC,MAAM,EAAE,UAAU,CAAC,CAAC;IAChD,QAAQ,CAAC,cAAc,EAAE,WAAW,CAAC,MAAM,EAAE,cAAc,CAAC,CAAC;IAC7D,QAAQ,CAAC,WAAW,EAAE,gBAAgB,CAAC;IACvC,QAAQ,CAAC,QAAQ,EAAE,WAAW,CAAC,MAAM,EAAE,SAAS,QAAQ,EAAE,CAAC,CAAC;CAC7D;AAED;;;;;;;GAOG;AACH,MAAM,WAAW,SAAS;IACxB,QAAQ,CAAC,sBAAsB,EAAE,MAAM,CAAC;IACxC,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;IAC9B,QAAQ,CAAC,kBAAkB,EAAE,MAAM,CAAC;IACpC,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,gBAAgB,EAAE,MAAM,CAAC;IAClC,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAC;IAChC,QAAQ,CAAC,gBAAgB,EAAE,MAAM,CAAC;IAClC,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAC;IAChC,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;CAC9B;AAED;;GAEG;AACH,eAAO,MAAM,QAAQ,EAAE,SAYtB,CAAC;AAEF;;GAEG;AACH,MAAM,WAAW,UAAU;IACzB,QAAQ,CAAC,KAAK,EAAE,eAAe,GAAG,SAAS,CAAC;IAC5C,8EAA8E;IAC9E,QAAQ,CAAC,QAAQ,EAAE,SAAS,MAAM,EAAE,GAAG,SAAS,CAAC;IACjD,QAAQ,CAAC,KAAK,EAAE,SAAS,CAAC;CAC3B;AAED;;GAEG;AACH,wBAAgB,QAAQ,CAAC,KAAK,EAAE,SAAS,EAAE,KAAK,EAAE,SAAS,GAAG,SAAS,CAetE"}
|
|
1
|
+
{"version":3,"file":"merge-types.d.ts","sourceRoot":"","sources":["../../src/dictionary/merge-types.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AACH,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,sBAAsB,CAAC;AACxD,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,qBAAqB,CAAC;AACtD,OAAO,KAAK,EAAE,cAAc,EAAE,cAAc,EAAE,MAAM,sBAAsB,CAAC;AAC3E,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,yBAAyB,CAAC;AACxD,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,YAAY,CAAC;AAClD,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,gBAAgB,CAAC;AACpD,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,wBAAwB,CAAC;AAE/D;;GAEG;AACH,MAAM,WAAW,YAAY;IAC3B,QAAQ,CAAC,cAAc,EAAE,WAAW,CAAC,MAAM,EAAE,cAAc,CAAC,CAAC;IAC7D,QAAQ,CAAC,cAAc,EAAE,cAAc,CAAC;IACxC,QAAQ,CAAC,MAAM,EAAE,WAAW,CAAC,MAAM,EAAE,SAAS,MAAM,EAAE,CAAC,CAAC;IACxD,QAAQ,CAAC,MAAM,EAAE,SAAS,WAAW,EAAE,CAAC;IACxC,QAAQ,CAAC,KAAK,EAAE,WAAW,CAAC,MAAM,EAAE,UAAU,CAAC,CAAC;CACjD;AAED;;GAEG;AACH,MAAM,WAAW,UAAU;IACzB,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,iEAAiE;IACjE,QAAQ,CAAC,sBAAsB,EAAE,MAAM,CAAC;IACxC;;;OAGG;IACH,QAAQ,CAAC,eAAe,EAAE,MAAM,CAAC;IACjC,qEAAqE;IACrE,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;IAC9B;;;OAGG;IACH,QAAQ,CAAC,mBAAmB,EAAE,MAAM,CAAC;IACrC,wEAAwE;IACxE,QAAQ,CAAC,kBAAkB,EAAE,MAAM,CAAC;IACpC,0DAA0D;IAC1D,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,4DAA4D;IAC5D,QAAQ,CAAC,gBAAgB,EAAE,MAAM,CAAC;IAClC;;;OAGG;IACH,QAAQ,CAAC,oBAAoB,EAAE,MAAM,CAAC;IACtC,8DAA8D;IAC9D,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAC;IAChC,sEAAsE;IACtE,QAAQ,CAAC,sBAAsB,EAAE,MAAM,CAAC;IACxC;;OAEG;IACH,QAAQ,CAAC,gBAAgB,EAAE,MAAM,CAAC;IAClC,8DAA8D;IAC9D,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAC;IAChC;;;OAGG;IACH,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,iEAAiE;IACjE,QAAQ,CAAC,aAAa,EAAE,MAAM,CAAC;IAC/B,oEAAoE;IACpE,QAAQ,CAAC,eAAe,EAAE,MAAM,CAAC;IACjC,6DAA6D;IAC7D,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;CAC3B;AAED;;GAEG;AACH,MAAM,WAAW,WAAW;IAC1B,yCAAyC;IACzC,QAAQ,CAAC,OAAO,EAAE,SAAS,eAAe,EAAE,CAAC;IAC7C,mEAAmE;IACnE,QAAQ,CAAC,QAAQ,EAAE,WAAW,CAAC,MAAM,EAAE,SAAS,MAAM,EAAE,CAAC,CAAC;IAC1D,QAAQ,CAAC,KAAK,EAAE,UAAU,CAAC;CAC5B;AAED;;;;;;GAMG;AACH,MAAM,WAAW,WAAW;IAC1B,QAAQ,CAAC,YAAY,EAAE,WAAW,CAAC,MAAM,EAAE,SAAS,WAAW,EAAE,CAAC,CAAC;IACnE,QAAQ,CAAC,YAAY,EAAE,WAAW,CAAC,MAAM,EAAE,SAAS,WAAW,EAAE,CAAC,CAAC;IACnE,QAAQ,CAAC,MAAM,EAAE,WAAW,CAAC,MAAM,EAAE,SAAS,MAAM,EAAE,CAAC,CAAC;IACxD,QAAQ,CAAC,KAAK,EAAE,WAAW,CAAC,MAAM,EAAE,UAAU,CAAC,CAAC;IAChD,2GAA2G;IAC3G,QAAQ,CAAC,QAAQ,EAAE,aAAa,CAAC;IACjC,QAAQ,CAAC,cAAc,EAAE,WAAW,CAAC,MAAM,EAAE,cAAc,CAAC,CAAC;IAC7D,QAAQ,CAAC,WAAW,EAAE,gBAAgB,CAAC;IACvC,QAAQ,CAAC,QAAQ,EAAE,WAAW,CAAC,MAAM,EAAE,SAAS,QAAQ,EAAE,CAAC,CAAC;CAC7D;AAED;;;;;;;GAOG;AACH,MAAM,WAAW,SAAS;IACxB,QAAQ,CAAC,sBAAsB,EAAE,MAAM,CAAC;IACxC,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;IAC9B,QAAQ,CAAC,kBAAkB,EAAE,MAAM,CAAC;IACpC,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,gBAAgB,EAAE,MAAM,CAAC;IAClC,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAC;IAChC,QAAQ,CAAC,gBAAgB,EAAE,MAAM,CAAC;IAClC,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAC;IAChC,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;CAC9B;AAED;;GAEG;AACH,eAAO,MAAM,QAAQ,EAAE,SAYtB,CAAC;AAEF;;GAEG;AACH,MAAM,WAAW,UAAU;IACzB,QAAQ,CAAC,KAAK,EAAE,eAAe,GAAG,SAAS,CAAC;IAC5C,8EAA8E;IAC9E,QAAQ,CAAC,QAAQ,EAAE,SAAS,MAAM,EAAE,GAAG,SAAS,CAAC;IACjD,QAAQ,CAAC,KAAK,EAAE,SAAS,CAAC;CAC3B;AAED;;GAEG;AACH,wBAAgB,QAAQ,CAAC,KAAK,EAAE,SAAS,EAAE,KAAK,EAAE,SAAS,GAAG,SAAS,CAetE"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"merge-types.js","sourceRoot":"","sources":["../../src/dictionary/merge-types.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"merge-types.js","sourceRoot":"","sources":["../../src/dictionary/merge-types.ts"],"names":[],"mappings":"AAoIA;;GAEG;AACH,MAAM,CAAC,MAAM,QAAQ,GAAc;IACjC,sBAAsB,EAAE,CAAC;IACzB,YAAY,EAAE,CAAC;IACf,kBAAkB,EAAE,CAAC;IACrB,WAAW,EAAE,CAAC;IACd,gBAAgB,EAAE,CAAC;IACnB,cAAc,EAAE,CAAC;IACjB,gBAAgB,EAAE,CAAC;IACnB,cAAc,EAAE,CAAC;IACjB,UAAU,EAAE,CAAC;IACb,WAAW,EAAE,CAAC;IACd,WAAW,EAAE,CAAC;CACf,CAAC;AAYF;;GAEG;AACH,MAAM,UAAU,QAAQ,CAAC,KAAgB,EAAE,KAAgB;IACzD,OAAO;QACL,sBAAsB,EACpB,KAAK,CAAC,sBAAsB,GAAG,KAAK,CAAC,sBAAsB;QAC7D,YAAY,EAAE,KAAK,CAAC,YAAY,GAAG,KAAK,CAAC,YAAY;QACrD,kBAAkB,EAAE,KAAK,CAAC,kBAAkB,GAAG,KAAK,CAAC,kBAAkB;QACvE,WAAW,EAAE,KAAK,CAAC,WAAW,GAAG,KAAK,CAAC,WAAW;QAClD,gBAAgB,EAAE,KAAK,CAAC,gBAAgB,GAAG,KAAK,CAAC,gBAAgB;QACjE,cAAc,EAAE,KAAK,CAAC,cAAc,GAAG,KAAK,CAAC,cAAc;QAC3D,gBAAgB,EAAE,KAAK,CAAC,gBAAgB,GAAG,KAAK,CAAC,gBAAgB;QACjE,cAAc,EAAE,KAAK,CAAC,cAAc,GAAG,KAAK,CAAC,cAAc;QAC3D,UAAU,EAAE,KAAK,CAAC,UAAU,GAAG,KAAK,CAAC,UAAU;QAC/C,WAAW,EAAE,KAAK,CAAC,WAAW,GAAG,KAAK,CAAC,WAAW;QAClD,WAAW,EAAE,KAAK,CAAC,WAAW,GAAG,KAAK,CAAC,WAAW;KACnD,CAAC;AACJ,CAAC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"merge-word.d.ts","sourceRoot":"","sources":["../../src/dictionary/merge-word.ts"],"names":[],"mappings":"AAgBA,OAAO,EAAE,KAAK,UAAU,EAAY,KAAK,WAAW,EAAE,MAAM,kBAAkB,CAAC;AAE/E,YAAY,EAAE,UAAU,EAAE,WAAW,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AAE3E;;GAEG;AACH,wBAAgB,SAAS,CAAC,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,WAAW,GAAG,UAAU,
|
|
1
|
+
{"version":3,"file":"merge-word.d.ts","sourceRoot":"","sources":["../../src/dictionary/merge-word.ts"],"names":[],"mappings":"AAgBA,OAAO,EAAE,KAAK,UAAU,EAAY,KAAK,WAAW,EAAE,MAAM,kBAAkB,CAAC;AAE/E,YAAY,EAAE,UAAU,EAAE,WAAW,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AAE3E;;GAEG;AACH,wBAAgB,SAAS,CAAC,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,WAAW,GAAG,UAAU,CAgHxE"}
|
|
@@ -19,7 +19,7 @@ import { NO_TALLY } from "./merge-types.js";
|
|
|
19
19
|
* Merge one word.
|
|
20
20
|
*/
|
|
21
21
|
export function mergeWord(word, sources) {
|
|
22
|
-
const { cedictByWord, cedictByHant, phrase, jieba, unihanReadings, traditional, defaults, } = sources;
|
|
22
|
+
const { cedictByWord, cedictByHant, phrase, jieba, nameMass, unihanReadings, traditional, defaults, } = sources;
|
|
23
23
|
const cedictEntries = cedictByWord.get(word) ?? [];
|
|
24
24
|
const cedictReadings = cedictReadingsOf(word, cedictEntries);
|
|
25
25
|
// Every reading CC-CEDICT gives this spelling, under either script.
|
|
@@ -47,7 +47,7 @@ export function mergeWord(word, sources) {
|
|
|
47
47
|
const taiwan = taiwanReadingOf(word, senses, senseReadings, reading, unihanReadings.get(word));
|
|
48
48
|
// ── Frequency, part of speech and the proper noun bit ─────
|
|
49
49
|
const jiebaEntry = jieba.get(word);
|
|
50
|
-
const { isProperNoun, boundaries, isVetoed } = properNounOf(word, jiebaEntry, cedictEntries, senses, reading);
|
|
50
|
+
const { isProperNoun, boundaries, isVetoed } = properNounOf(word, jiebaEntry, cedictEntries, senses, reading, nameMass);
|
|
51
51
|
// ── Polyphone priors, for single characters only ──────────
|
|
52
52
|
const isCharacter = isSingleCharacter(word);
|
|
53
53
|
const characterReadings = defaults.get(word) ?? [];
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"merge-word.js","sourceRoot":"","sources":["../../src/dictionary/merge-word.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AACH,OAAO,EAAE,iBAAiB,EAAE,MAAM,yBAAyB,CAAC;AAC5D,OAAO,EAAwB,aAAa,EAAE,MAAM,YAAY,CAAC;AACjE,OAAO,EAAE,kBAAkB,EAAE,MAAM,cAAc,CAAC;AAClD,OAAO,EAAE,gBAAgB,EAAE,MAAM,oBAAoB,CAAC;AACtD,OAAO,EAAE,iBAAiB,EAAE,MAAM,uBAAuB,CAAC;AAC1D,OAAO,EAAE,eAAe,EAAE,MAAM,qBAAqB,CAAC;AACtD,OAAO,EAAE,YAAY,EAAE,MAAM,kBAAkB,CAAC;AAChD,OAAO,EAAE,aAAa,EAAE,MAAM,mBAAmB,CAAC;AAClD,OAAO,EAAmB,QAAQ,EAAoB,MAAM,kBAAkB,CAAC;AAI/E;;GAEG;AACH,MAAM,UAAU,SAAS,CAAC,IAAY,EAAE,OAAoB;IAC1D,MAAM,EACJ,YAAY,EACZ,YAAY,EACZ,MAAM,EACN,KAAK,EACL,cAAc,EACd,WAAW,EACX,QAAQ,GACT,GAAG,OAAO,CAAC;IAEZ,MAAM,aAAa,GAAG,YAAY,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC;IACnD,MAAM,cAAc,GAAG,gBAAgB,CAAC,IAAI,EAAE,aAAa,CAAC,CAAC;IAC7D,oEAAoE;IACpE,MAAM,aAAa,GAAG;QACpB,GAAG,cAAc;QACjB,GAAG,gBAAgB,CAAC,IAAI,EAAE,YAAY,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC;KACxD,CAAC;IACF,MAAM,aAAa,GAAG,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;IACvC,MAAM,aAAa,GACjB,aAAa,KAAK,SAAS;QACzB,CAAC,CAAC,SAAS;QACX,CAAC,CAAC,kBAAkB,CAAC,IAAI,EAAE,aAAa,CAAC,CAAC;IAE9C,MAAM,OAAO,GAAG,aAAa,CAC3B,IAAI,EACJ,QAAQ,EACR,aAAa,EACb,cAAc,EACd,aAAa,CACd,CAAC;IACF,IAAI,OAAO,CAAC,OAAO,KAAK,SAAS,EAAE,CAAC;QAClC,OAAO;YACL,KAAK,EAAE,SAAS;YAChB,QAAQ,EAAE,aAAa,IAAI,aAAa,CAAC,CAAC,CAAC,EAAE,QAAQ,IAAI,EAAE;YAC3D,KAAK,EAAE,QAAQ;SAChB,CAAC;IACJ,CAAC;IACD,MAAM,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC;IAChC,MAAM,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC;IAEhC,4DAA4D;IAC5D,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,YAAY,EAAE,SAAS,EAAE,GAAG,iBAAiB,CACjE,IAAI,EACJ,aAAa,EACb,OAAO,EACP,OAAO,EACP,WAAW,CACZ,CAAC;IAEF,6DAA6D;IAC7D,MAAM,MAAM,GAAG,eAAe,CAC5B,IAAI,EACJ,MAAM,EACN,aAAa,EACb,OAAO,EACP,cAAc,CAAC,GAAG,CAAC,IAAI,CAAC,CACzB,CAAC;IAEF,6DAA6D;IAC7D,MAAM,UAAU,GAAG,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;IACnC,MAAM,EAAE,YAAY,EAAE,UAAU,EAAE,QAAQ,EAAE,GAAG,YAAY,CACzD,IAAI,EACJ,UAAU,EACV,aAAa,EACb,MAAM,EACN,OAAO,
|
|
1
|
+
{"version":3,"file":"merge-word.js","sourceRoot":"","sources":["../../src/dictionary/merge-word.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AACH,OAAO,EAAE,iBAAiB,EAAE,MAAM,yBAAyB,CAAC;AAC5D,OAAO,EAAwB,aAAa,EAAE,MAAM,YAAY,CAAC;AACjE,OAAO,EAAE,kBAAkB,EAAE,MAAM,cAAc,CAAC;AAClD,OAAO,EAAE,gBAAgB,EAAE,MAAM,oBAAoB,CAAC;AACtD,OAAO,EAAE,iBAAiB,EAAE,MAAM,uBAAuB,CAAC;AAC1D,OAAO,EAAE,eAAe,EAAE,MAAM,qBAAqB,CAAC;AACtD,OAAO,EAAE,YAAY,EAAE,MAAM,kBAAkB,CAAC;AAChD,OAAO,EAAE,aAAa,EAAE,MAAM,mBAAmB,CAAC;AAClD,OAAO,EAAmB,QAAQ,EAAoB,MAAM,kBAAkB,CAAC;AAI/E;;GAEG;AACH,MAAM,UAAU,SAAS,CAAC,IAAY,EAAE,OAAoB;IAC1D,MAAM,EACJ,YAAY,EACZ,YAAY,EACZ,MAAM,EACN,KAAK,EACL,QAAQ,EACR,cAAc,EACd,WAAW,EACX,QAAQ,GACT,GAAG,OAAO,CAAC;IAEZ,MAAM,aAAa,GAAG,YAAY,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC;IACnD,MAAM,cAAc,GAAG,gBAAgB,CAAC,IAAI,EAAE,aAAa,CAAC,CAAC;IAC7D,oEAAoE;IACpE,MAAM,aAAa,GAAG;QACpB,GAAG,cAAc;QACjB,GAAG,gBAAgB,CAAC,IAAI,EAAE,YAAY,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC;KACxD,CAAC;IACF,MAAM,aAAa,GAAG,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;IACvC,MAAM,aAAa,GACjB,aAAa,KAAK,SAAS;QACzB,CAAC,CAAC,SAAS;QACX,CAAC,CAAC,kBAAkB,CAAC,IAAI,EAAE,aAAa,CAAC,CAAC;IAE9C,MAAM,OAAO,GAAG,aAAa,CAC3B,IAAI,EACJ,QAAQ,EACR,aAAa,EACb,cAAc,EACd,aAAa,CACd,CAAC;IACF,IAAI,OAAO,CAAC,OAAO,KAAK,SAAS,EAAE,CAAC;QAClC,OAAO;YACL,KAAK,EAAE,SAAS;YAChB,QAAQ,EAAE,aAAa,IAAI,aAAa,CAAC,CAAC,CAAC,EAAE,QAAQ,IAAI,EAAE;YAC3D,KAAK,EAAE,QAAQ;SAChB,CAAC;IACJ,CAAC;IACD,MAAM,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC;IAChC,MAAM,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC;IAEhC,4DAA4D;IAC5D,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,YAAY,EAAE,SAAS,EAAE,GAAG,iBAAiB,CACjE,IAAI,EACJ,aAAa,EACb,OAAO,EACP,OAAO,EACP,WAAW,CACZ,CAAC;IAEF,6DAA6D;IAC7D,MAAM,MAAM,GAAG,eAAe,CAC5B,IAAI,EACJ,MAAM,EACN,aAAa,EACb,OAAO,EACP,cAAc,CAAC,GAAG,CAAC,IAAI,CAAC,CACzB,CAAC;IAEF,6DAA6D;IAC7D,MAAM,UAAU,GAAG,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;IACnC,MAAM,EAAE,YAAY,EAAE,UAAU,EAAE,QAAQ,EAAE,GAAG,YAAY,CACzD,IAAI,EACJ,UAAU,EACV,aAAa,EACb,MAAM,EACN,OAAO,EACP,QAAQ,CACT,CAAC;IAEF,6DAA6D;IAC7D,MAAM,WAAW,GAAG,iBAAiB,CAAC,IAAI,CAAC,CAAC;IAC5C,MAAM,iBAAiB,GAAG,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC;IACnD,MAAM,UAAU,GAAG,WAAW;QAC5B,CAAC,CAAC,iBAAiB;aACd,MAAM,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC,aAAa,CAAC,CAAC,QAAQ,CAAC,EAAE,OAAO,CAAC,CAAC;aACzD,GAAG,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC,QAAQ,CAAC,CAAC;QAClC,CAAC,CAAC,EAAE,CAAC;IAEP,MAAM,KAAK,GAAoB;QAC7B,IAAI,EAAE,IAAI;QACV,IAAI;QACJ,GAAG,CAAC,YAAY,CAAC,MAAM,GAAG,CAAC,IAAI,EAAE,YAAY,EAAE,CAAC;QAChD,QAAQ,EAAE,EAAE,EAAE,EAAE,OAAO,EAAE,GAAG,CAAC,MAAM,KAAK,SAAS,IAAI,EAAE,EAAE,EAAE,MAAM,EAAE,CAAC,EAAE;QACtE,SAAS,EAAE,UAAU,EAAE,SAAS,IAAI,CAAC;QACrC,YAAY,EAAE,UAAU,EAAE,YAAY,IAAI,EAAE;QAC5C,YAAY;QACZ,GAAG,CAAC,UAAU,CAAC,MAAM,GAAG,CAAC,IAAI,EAAE,cAAc,EAAE,UAAU,EAAE,CAAC;QAC5D,GAAG,CAAC,UAAU,CAAC,MAAM,GAAG,CAAC,IAAI,EAAE,UAAU,EAAE,CAAC;KAC7C,CAAC;IAEF,4EAA4E;IAC5E,4EAA4E;IAC5E,gDAAgD;IAChD,OAAO;QACL,KAAK;QACL,QAAQ,EAAE,SAAS;QACnB,KAAK,EAAE;YACL,sBAAsB,EAAE,OAAO,CAAC,sBAAsB;YACtD,YAAY,EAAE,OAAO,CAAC,YAAY;YAClC,kBAAkB,EAAE,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YACrC,WAAW,EAAE,IAAI,KAAK,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YAClC,gBAAgB,EAAE,YAAY,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YACjD,cAAc,EAAE,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YAC5C,gBAAgB,EAAE,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YAClC,cAAc,EAAE,UAAU,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YAC7C,UAAU,EAAE,WAAW,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YAC/B,WAAW,EAAE,aAAa,KAAK,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YAChD,WAAW,EAAE,aAAa,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;SAC9C;KACF,CAAC;AACJ,CAAC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"merge.d.ts","sourceRoot":"","sources":["../../src/dictionary/merge.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"merge.d.ts","sourceRoot":"","sources":["../../src/dictionary/merge.ts"],"names":[],"mappings":"AAeA,OAAO,KAAK,EAAE,WAAW,EAAE,YAAY,EAAe,MAAM,kBAAkB,CAAC;AAE/E,YAAY,EAAE,WAAW,EAAE,YAAY,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAC;AAE9E;;;;;;;;GAQG;AACH,wBAAgB,YAAY,CAAC,OAAO,EAAE,YAAY,GAAG,WAAW,CAsE/D"}
|
package/dist/dictionary/merge.js
CHANGED
|
@@ -11,6 +11,8 @@ import { buildCharacterDefaults } from "./character-defaults.js";
|
|
|
11
11
|
import { repairConstituentReadings } from "./constituent-repair.js";
|
|
12
12
|
import { settleMeasureTones } from "./measure-tone.js";
|
|
13
13
|
import { mergeWords, wordList } from "./merge-walk.js";
|
|
14
|
+
import { countNameMass } from "./name-mass.js";
|
|
15
|
+
import { carryToTraditional } from "./traditional-carry.js";
|
|
14
16
|
/**
|
|
15
17
|
* Build the merged dictionary from the four parsed sources.
|
|
16
18
|
*
|
|
@@ -34,6 +36,7 @@ export function mergeSources(sources) {
|
|
|
34
36
|
cedictByHant,
|
|
35
37
|
phrase,
|
|
36
38
|
jieba: sources.jieba,
|
|
39
|
+
nameMass: countNameMass(sources.jieba),
|
|
37
40
|
unihanReadings: sources.unihanReadings,
|
|
38
41
|
traditional,
|
|
39
42
|
defaults,
|
|
@@ -53,8 +56,12 @@ export function mergeSources(sources) {
|
|
|
53
56
|
// readings, both scripts' keys and the frequencies all have to be settled
|
|
54
57
|
// before a compound can be asked what it is made of.
|
|
55
58
|
const localised = composeLocaleDeltas(held.entries);
|
|
59
|
+
// ── 繁體 spellings given their 简体 word's tag and count ────
|
|
60
|
+
// Last, and over the finished key set, because the pairing it reads is the
|
|
61
|
+
// 繁體 form each entry names and every pass above can still change one.
|
|
62
|
+
const carried = carryToTraditional(localised.entries);
|
|
56
63
|
return {
|
|
57
|
-
entries:
|
|
64
|
+
entries: carried.entries,
|
|
58
65
|
rejected,
|
|
59
66
|
stats: {
|
|
60
67
|
...counts,
|
|
@@ -62,6 +69,9 @@ export function mergeSources(sources) {
|
|
|
62
69
|
settledMeasureTones: measured.settled,
|
|
63
70
|
repairedConstituents: held.repaired,
|
|
64
71
|
composedTaiwanReadings: localised.composed,
|
|
72
|
+
carriedTags: carried.carriedTags,
|
|
73
|
+
carriedCounts: carried.carriedCounts,
|
|
74
|
+
carriedCapitals: carried.carriedCapitals,
|
|
65
75
|
rejected: rejected.size,
|
|
66
76
|
},
|
|
67
77
|
};
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"merge.js","sourceRoot":"","sources":["../../src/dictionary/merge.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AACH,OAAO,EAAE,mBAAmB,EAAE,MAAM,aAAa,CAAC;AAClD,OAAO,EAAE,WAAW,EAAE,WAAW,EAAE,MAAM,oBAAoB,CAAC;AAC9D,OAAO,EAAE,sBAAsB,EAAE,MAAM,yBAAyB,CAAC;AACjE,OAAO,EAAE,yBAAyB,EAAE,MAAM,yBAAyB,CAAC;AACpE,OAAO,EAAE,kBAAkB,EAAE,MAAM,mBAAmB,CAAC;AACvD,OAAO,EAAE,UAAU,EAAE,QAAQ,EAAE,MAAM,iBAAiB,CAAC;
|
|
1
|
+
{"version":3,"file":"merge.js","sourceRoot":"","sources":["../../src/dictionary/merge.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AACH,OAAO,EAAE,mBAAmB,EAAE,MAAM,aAAa,CAAC;AAClD,OAAO,EAAE,WAAW,EAAE,WAAW,EAAE,MAAM,oBAAoB,CAAC;AAC9D,OAAO,EAAE,sBAAsB,EAAE,MAAM,yBAAyB,CAAC;AACjE,OAAO,EAAE,yBAAyB,EAAE,MAAM,yBAAyB,CAAC;AACpE,OAAO,EAAE,kBAAkB,EAAE,MAAM,mBAAmB,CAAC;AACvD,OAAO,EAAE,UAAU,EAAE,QAAQ,EAAE,MAAM,iBAAiB,CAAC;AACvD,OAAO,EAAE,aAAa,EAAE,MAAM,gBAAgB,CAAC;AAC/C,OAAO,EAAE,kBAAkB,EAAE,MAAM,wBAAwB,CAAC;AAK5D;;;;;;;;GAQG;AACH,MAAM,UAAU,YAAY,CAAC,OAAqB;IAChD,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC;IACnC,MAAM,YAAY,GAAG,WAAW,CAAC,MAAM,EAAE,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;IACtE,6EAA6E;IAC7E,2EAA2E;IAC3E,0EAA0E;IAC1E,kCAAkC;IAClC,MAAM,YAAY,GAAG,WAAW,CAAC,MAAM,EAAE,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC;IAEvE,MAAM,EAAE,WAAW,EAAE,QAAQ,EAAE,eAAe,EAAE,GAAG,sBAAsB,CACvE,OAAO,EACP,YAAY,EACZ,YAAY,CACb,CAAC;IAEF,MAAM,WAAW,GAAgB;QAC/B,YAAY;QACZ,YAAY;QACZ,MAAM;QACN,KAAK,EAAE,OAAO,CAAC,KAAK;QACpB,QAAQ,EAAE,aAAa,CAAC,OAAO,CAAC,KAAK,CAAC;QACtC,cAAc,EAAE,OAAO,CAAC,cAAc;QACtC,WAAW;QACX,QAAQ;KACT,CAAC;IACF,MAAM,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM,EAAE,GAAG,UAAU,CAC9C,QAAQ,CAAC,MAAM,EAAE,QAAQ,EAAE,YAAY,EAAE,YAAY,CAAC,EACtD,WAAW,CACZ,CAAC;IAEF,6DAA6D;IAC7D,8EAA8E;IAC9E,oEAAoE;IACpE,MAAM,QAAQ,GAAG,kBAAkB,CAAC,OAAO,CAAC,CAAC;IAE7C,8DAA8D;IAC9D,4EAA4E;IAC5E,8EAA8E;IAC9E,0CAA0C;IAC1C,MAAM,IAAI,GAAG,yBAAyB,CACpC,QAAQ,CAAC,OAAO,EAChB,WAAW,CAAC,YAAY,EAAE,YAAY,CAAC,CACxC,CAAC;IAEF,8DAA8D;IAC9D,4EAA4E;IAC5E,0EAA0E;IAC1E,qDAAqD;IACrD,MAAM,SAAS,GAAG,mBAAmB,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IAEpD,2DAA2D;IAC3D,2EAA2E;IAC3E,sEAAsE;IACtE,MAAM,OAAO,GAAG,kBAAkB,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC;IAEtD,OAAO;QACL,OAAO,EAAE,OAAO,CAAC,OAAO;QACxB,QAAQ;QACR,KAAK,EAAE;YACL,GAAG,MAAM;YACT,eAAe;YACf,mBAAmB,EAAE,QAAQ,CAAC,OAAO;YACrC,oBAAoB,EAAE,IAAI,CAAC,QAAQ;YACnC,sBAAsB,EAAE,SAAS,CAAC,QAAQ;YAC1C,WAAW,EAAE,OAAO,CAAC,WAAW;YAChC,aAAa,EAAE,OAAO,CAAC,aAAa;YACpC,eAAe,EAAE,OAAO,CAAC,eAAe;YACxC,QAAQ,EAAE,QAAQ,CAAC,IAAI;SACxB;KACF,CAAC;AACJ,CAAC"}
|
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
import { type JiebaEntry } from "../sources/jieba.js";
|
|
2
|
+
/**
|
|
3
|
+
* The corpus mass of the words one character begins, split by what they are.
|
|
4
|
+
*/
|
|
5
|
+
export interface NameMass {
|
|
6
|
+
/** Occurrences of the names it begins. */
|
|
7
|
+
readonly asName: number;
|
|
8
|
+
/** Occurrences of the ordinary words it begins. */
|
|
9
|
+
readonly asWord: number;
|
|
10
|
+
}
|
|
11
|
+
/**
|
|
12
|
+
* What each character heads, keyed by the character.
|
|
13
|
+
*/
|
|
14
|
+
export type NameMassTable = ReadonlyMap<string, NameMass>;
|
|
15
|
+
/**
|
|
16
|
+
* Count what every character heads in jieba's dictionary.
|
|
17
|
+
*
|
|
18
|
+
* **Only the first character of a word counts.** A capital opens a name and the
|
|
19
|
+
* question here is what opens one, so 德 is counted for 德国 and not for 伍德.
|
|
20
|
+
* Counting the inside of a name too would say that 德 is a name character,
|
|
21
|
+
* which is true and answers a different question from the one the capital asks.
|
|
22
|
+
*
|
|
23
|
+
* Single characters are skipped, since a character heading itself would only
|
|
24
|
+
* confirm whatever tag it already carries.
|
|
25
|
+
*/
|
|
26
|
+
export declare function countNameMass(jieba: ReadonlyMap<string, JiebaEntry>): NameMassTable;
|
|
27
|
+
/**
|
|
28
|
+
* Whether jieba's corpus met a character at the head of a name more often than
|
|
29
|
+
* as a word.
|
|
30
|
+
*
|
|
31
|
+
* The character's own count is on the side of the words, because a bare
|
|
32
|
+
* character in a 简体 corpus is a word being used as one. 帅 heads 155
|
|
33
|
+
* occurrences of names against 86 of words, which would pass, and it was
|
|
34
|
+
* counted 795 times on its own reading `shuài` and meaning handsome, which
|
|
35
|
+
* settles it the other way.
|
|
36
|
+
*
|
|
37
|
+
* A character jieba never saw heading a word has nothing on either side and
|
|
38
|
+
* fails. What that costs is a 繁體 character, whose 简体 pairing lends it the
|
|
39
|
+
* answer afterwards in `traditional-carry.ts`.
|
|
40
|
+
*/
|
|
41
|
+
export declare function leadsNames(mass: NameMassTable, character: string, frequency: number): boolean;
|
|
42
|
+
//# sourceMappingURL=name-mass.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"name-mass.d.ts","sourceRoot":"","sources":["../../src/dictionary/name-mass.ts"],"names":[],"mappings":"AAoBA,OAAO,EAAmB,KAAK,UAAU,EAAE,MAAM,qBAAqB,CAAC;AAEvE;;GAEG;AACH,MAAM,WAAW,QAAQ;IACvB,0CAA0C;IAC1C,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,mDAAmD;IACnD,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;CACzB;AAED;;GAEG;AACH,MAAM,MAAM,aAAa,GAAG,WAAW,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;AAE1D;;;;;;;;;;GAUG;AACH,wBAAgB,aAAa,CAC3B,KAAK,EAAE,WAAW,CAAC,MAAM,EAAE,UAAU,CAAC,GACrC,aAAa,CAmBf;AAED;;;;;;;;;;;;;GAaG;AACH,wBAAgB,UAAU,CACxB,IAAI,EAAE,aAAa,EACnB,SAAS,EAAE,MAAM,EACjB,SAAS,EAAE,MAAM,GAChB,OAAO,CAGT"}
|
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* How much of jieba's corpus a character carries at the head of a name.
|
|
3
|
+
*
|
|
4
|
+
* A word is usually one thing, and 北京 being a place in its only sense settles
|
|
5
|
+
* what 北京 is. A character is a morpheme that turns up in hundreds of words,
|
|
6
|
+
* and nearly every character that is a surname is an ordinary word besides. 王
|
|
7
|
+
* is `Wáng` and `wáng` the king, 连 is `Lián` and "even", 云 is `Yún` and a
|
|
8
|
+
* cloud. Neither source settles that on its own. jieba tags the bare character
|
|
9
|
+
* `nr` for all three, and CC-CEDICT gives all three a capitalised sense and a
|
|
10
|
+
* lowercase one.
|
|
11
|
+
*
|
|
12
|
+
* What separates them is how often each is met as a name, and jieba's own word
|
|
13
|
+
* list carries it. The character heads words there, each with a corpus count
|
|
14
|
+
* and a tag of its own. 李 heads 40,346 occurrences of names against 9,787 of
|
|
15
|
+
* ordinary words, and 连 heads 2,102 against 23,042 of 连续, 连接 and 连忙.
|
|
16
|
+
* Counted this way the surnames separate from the words that happen to be
|
|
17
|
+
* surnames, and the list that comes out is the one a reader would write down:
|
|
18
|
+
* 陈, 赵, 杨, 吴, 朱, 郭, 蒋, 蔡 and about a hundred others.
|
|
19
|
+
*/
|
|
20
|
+
import { toCharacters } from "../script/characters.js";
|
|
21
|
+
import { isProperNounTag } from "../sources/jieba.js";
|
|
22
|
+
/**
|
|
23
|
+
* Count what every character heads in jieba's dictionary.
|
|
24
|
+
*
|
|
25
|
+
* **Only the first character of a word counts.** A capital opens a name and the
|
|
26
|
+
* question here is what opens one, so 德 is counted for 德国 and not for 伍德.
|
|
27
|
+
* Counting the inside of a name too would say that 德 is a name character,
|
|
28
|
+
* which is true and answers a different question from the one the capital asks.
|
|
29
|
+
*
|
|
30
|
+
* Single characters are skipped, since a character heading itself would only
|
|
31
|
+
* confirm whatever tag it already carries.
|
|
32
|
+
*/
|
|
33
|
+
export function countNameMass(jieba) {
|
|
34
|
+
const mass = new Map();
|
|
35
|
+
for (const [word, entry] of jieba) {
|
|
36
|
+
const characters = toCharacters(word);
|
|
37
|
+
const head = characters[0];
|
|
38
|
+
if (characters.length < 2 || head === undefined) {
|
|
39
|
+
continue;
|
|
40
|
+
}
|
|
41
|
+
const held = mass.get(head) ?? { asName: 0, asWord: 0 };
|
|
42
|
+
mass.set(head, isProperNounTag(entry.partOfSpeech)
|
|
43
|
+
? { asName: held.asName + entry.frequency, asWord: held.asWord }
|
|
44
|
+
: { asName: held.asName, asWord: held.asWord + entry.frequency });
|
|
45
|
+
}
|
|
46
|
+
return mass;
|
|
47
|
+
}
|
|
48
|
+
/**
|
|
49
|
+
* Whether jieba's corpus met a character at the head of a name more often than
|
|
50
|
+
* as a word.
|
|
51
|
+
*
|
|
52
|
+
* The character's own count is on the side of the words, because a bare
|
|
53
|
+
* character in a 简体 corpus is a word being used as one. 帅 heads 155
|
|
54
|
+
* occurrences of names against 86 of words, which would pass, and it was
|
|
55
|
+
* counted 795 times on its own reading `shuài` and meaning handsome, which
|
|
56
|
+
* settles it the other way.
|
|
57
|
+
*
|
|
58
|
+
* A character jieba never saw heading a word has nothing on either side and
|
|
59
|
+
* fails. What that costs is a 繁體 character, whose 简体 pairing lends it the
|
|
60
|
+
* answer afterwards in `traditional-carry.ts`.
|
|
61
|
+
*/
|
|
62
|
+
export function leadsNames(mass, character, frequency) {
|
|
63
|
+
const held = mass.get(character);
|
|
64
|
+
return held !== undefined && held.asName > held.asWord + frequency;
|
|
65
|
+
}
|
|
66
|
+
//# sourceMappingURL=name-mass.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"name-mass.js","sourceRoot":"","sources":["../../src/dictionary/name-mass.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;GAkBG;AACH,OAAO,EAAE,YAAY,EAAE,MAAM,yBAAyB,CAAC;AACvD,OAAO,EAAE,eAAe,EAAmB,MAAM,qBAAqB,CAAC;AAiBvE;;;;;;;;;;GAUG;AACH,MAAM,UAAU,aAAa,CAC3B,KAAsC;IAEtC,MAAM,IAAI,GAAG,IAAI,GAAG,EAAoB,CAAC;IAEzC,KAAK,MAAM,CAAC,IAAI,EAAE,KAAK,CAAC,IAAI,KAAK,EAAE,CAAC;QAClC,MAAM,UAAU,GAAG,YAAY,CAAC,IAAI,CAAC,CAAC;QACtC,MAAM,IAAI,GAAG,UAAU,CAAC,CAAC,CAAC,CAAC;QAC3B,IAAI,UAAU,CAAC,MAAM,GAAG,CAAC,IAAI,IAAI,KAAK,SAAS,EAAE,CAAC;YAChD,SAAS;QACX,CAAC;QACD,MAAM,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,EAAE,MAAM,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,CAAC;QACxD,IAAI,CAAC,GAAG,CACN,IAAI,EACJ,eAAe,CAAC,KAAK,CAAC,YAAY,CAAC;YACjC,CAAC,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,GAAG,KAAK,CAAC,SAAS,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,EAAE;YAChE,CAAC,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,GAAG,KAAK,CAAC,SAAS,EAAE,CACnE,CAAC;IACJ,CAAC;IAED,OAAO,IAAI,CAAC;AACd,CAAC;AAED;;;;;;;;;;;;;GAaG;AACH,MAAM,UAAU,UAAU,CACxB,IAAmB,EACnB,SAAiB,EACjB,SAAiB;IAEjB,MAAM,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;IACjC,OAAO,IAAI,KAAK,SAAS,IAAI,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,GAAG,SAAS,CAAC;AACrE,CAAC"}
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import { type CedictEntry } from "../sources/cedict.js";
|
|
2
2
|
import { type JiebaEntry } from "../sources/jieba.js";
|
|
3
3
|
import type { Syllable } from "../syllable/syllable.js";
|
|
4
|
+
import { type NameMassTable } from "./name-mass.js";
|
|
4
5
|
/**
|
|
5
6
|
* What the sources between them say about a word's name-hood.
|
|
6
7
|
*/
|
|
@@ -23,6 +24,12 @@ export interface ProperNoun {
|
|
|
23
24
|
* The boundary is only looked for where the word survived the veto and reads
|
|
24
25
|
* one syllable per character: 儿化 reads two characters as one syllable and
|
|
25
26
|
* could not be cut by a count of characters.
|
|
27
|
+
*
|
|
28
|
+
* **A single character has to lead names as well**, which neither source can
|
|
29
|
+
* say on its own and jieba's word list can. See
|
|
30
|
+
* {@link import("./name-mass.js").leadsNames}. 411 characters carried the bit
|
|
31
|
+
* on one capitalised sense, and 他连再见也不说 came out `tā Lián zàijiàn yě bù
|
|
32
|
+
* shuō`.
|
|
26
33
|
*/
|
|
27
|
-
export declare function properNounOf(word: string, jiebaEntry: JiebaEntry | undefined, cedictEntries: readonly CedictEntry[], senses: readonly CedictEntry[], reading: readonly Syllable[]): ProperNoun;
|
|
34
|
+
export declare function properNounOf(word: string, jiebaEntry: JiebaEntry | undefined, cedictEntries: readonly CedictEntry[], senses: readonly CedictEntry[], reading: readonly Syllable[], nameMass: NameMassTable): ProperNoun;
|
|
28
35
|
//# sourceMappingURL=proper-noun.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"proper-noun.d.ts","sourceRoot":"","sources":["../../src/dictionary/proper-noun.ts"],"names":[],"mappings":"AASA,OAAO,EAAE,KAAK,WAAW,EAAoB,MAAM,sBAAsB,CAAC;AAC1E,OAAO,EAAmB,KAAK,UAAU,EAAE,MAAM,qBAAqB,CAAC;AACvE,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,yBAAyB,CAAC;
|
|
1
|
+
{"version":3,"file":"proper-noun.d.ts","sourceRoot":"","sources":["../../src/dictionary/proper-noun.ts"],"names":[],"mappings":"AASA,OAAO,EAAE,KAAK,WAAW,EAAoB,MAAM,sBAAsB,CAAC;AAC1E,OAAO,EAAmB,KAAK,UAAU,EAAE,MAAM,qBAAqB,CAAC;AACvE,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,yBAAyB,CAAC;AACxD,OAAO,EAAc,KAAK,aAAa,EAAE,MAAM,gBAAgB,CAAC;AAEhE;;GAEG;AACH,MAAM,WAAW,UAAU;IACzB,QAAQ,CAAC,YAAY,EAAE,OAAO,CAAC;IAC/B,oEAAoE;IACpE,QAAQ,CAAC,UAAU,EAAE,SAAS,MAAM,EAAE,CAAC;IACvC,kEAAkE;IAClE,QAAQ,CAAC,QAAQ,EAAE,OAAO,CAAC;CAC5B;AAED;;;;;;;;;;;;;;;;;;GAkBG;AACH,wBAAgB,YAAY,CAC1B,IAAI,EAAE,MAAM,EACZ,UAAU,EAAE,UAAU,GAAG,SAAS,EAClC,aAAa,EAAE,SAAS,WAAW,EAAE,EACrC,MAAM,EAAE,SAAS,WAAW,EAAE,EAC9B,OAAO,EAAE,SAAS,QAAQ,EAAE,EAC5B,QAAQ,EAAE,aAAa,GACtB,UAAU,CAuBZ"}
|
|
@@ -6,9 +6,10 @@
|
|
|
6
6
|
* 城市, 阿姨, 智能卡 and 花生仁 are all tagged nr or nz, and the decoder
|
|
7
7
|
* capitalises straight off this bit.
|
|
8
8
|
*/
|
|
9
|
-
import { characterCount } from "../script/characters.js";
|
|
9
|
+
import { characterCount, isSingleCharacter } from "../script/characters.js";
|
|
10
10
|
import { nameBoundariesOf } from "../sources/cedict.js";
|
|
11
11
|
import { isProperNounTag } from "../sources/jieba.js";
|
|
12
|
+
import { leadsNames } from "./name-mass.js";
|
|
12
13
|
/**
|
|
13
14
|
* Settle it for one word.
|
|
14
15
|
*
|
|
@@ -21,13 +22,22 @@ import { isProperNounTag } from "../sources/jieba.js";
|
|
|
21
22
|
* The boundary is only looked for where the word survived the veto and reads
|
|
22
23
|
* one syllable per character: 儿化 reads two characters as one syllable and
|
|
23
24
|
* could not be cut by a count of characters.
|
|
25
|
+
*
|
|
26
|
+
* **A single character has to lead names as well**, which neither source can
|
|
27
|
+
* say on its own and jieba's word list can. See
|
|
28
|
+
* {@link import("./name-mass.js").leadsNames}. 411 characters carried the bit
|
|
29
|
+
* on one capitalised sense, and 他连再见也不说 came out `tā Lián zàijiàn yě bù
|
|
30
|
+
* shuō`.
|
|
24
31
|
*/
|
|
25
|
-
export function properNounOf(word, jiebaEntry, cedictEntries, senses, reading) {
|
|
32
|
+
export function properNounOf(word, jiebaEntry, cedictEntries, senses, reading, nameMass) {
|
|
26
33
|
const partOfSpeech = jiebaEntry?.partOfSpeech ?? "";
|
|
27
|
-
const
|
|
34
|
+
const proposed = jiebaEntry === undefined
|
|
28
35
|
? cedictEntries.some((entry) => entry.isProperNoun)
|
|
29
36
|
: isProperNounTag(partOfSpeech) &&
|
|
30
37
|
(senses.length === 0 || senses.some((entry) => entry.isProperNoun));
|
|
38
|
+
const isProperNoun = proposed &&
|
|
39
|
+
(!isSingleCharacter(word) ||
|
|
40
|
+
leadsNames(nameMass, word, jiebaEntry?.frequency ?? 0));
|
|
31
41
|
const isAligned = characterCount(word) === reading.length;
|
|
32
42
|
const boundaries = isProperNoun && isAligned
|
|
33
43
|
? (cedictEntries
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"proper-noun.js","sourceRoot":"","sources":["../../src/dictionary/proper-noun.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AACH,OAAO,EAAE,cAAc,EAAE,MAAM,yBAAyB,CAAC;
|
|
1
|
+
{"version":3,"file":"proper-noun.js","sourceRoot":"","sources":["../../src/dictionary/proper-noun.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AACH,OAAO,EAAE,cAAc,EAAE,iBAAiB,EAAE,MAAM,yBAAyB,CAAC;AAC5E,OAAO,EAAoB,gBAAgB,EAAE,MAAM,sBAAsB,CAAC;AAC1E,OAAO,EAAE,eAAe,EAAmB,MAAM,qBAAqB,CAAC;AAEvE,OAAO,EAAE,UAAU,EAAsB,MAAM,gBAAgB,CAAC;AAahE;;;;;;;;;;;;;;;;;;GAkBG;AACH,MAAM,UAAU,YAAY,CAC1B,IAAY,EACZ,UAAkC,EAClC,aAAqC,EACrC,MAA8B,EAC9B,OAA4B,EAC5B,QAAuB;IAEvB,MAAM,YAAY,GAAG,UAAU,EAAE,YAAY,IAAI,EAAE,CAAC;IACpD,MAAM,QAAQ,GACZ,UAAU,KAAK,SAAS;QACtB,CAAC,CAAC,aAAa,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,YAAY,CAAC;QACnD,CAAC,CAAC,eAAe,CAAC,YAAY,CAAC;YAC7B,CAAC,MAAM,CAAC,MAAM,KAAK,CAAC,IAAI,MAAM,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,YAAY,CAAC,CAAC,CAAC;IAC1E,MAAM,YAAY,GAChB,QAAQ;QACR,CAAC,CAAC,iBAAiB,CAAC,IAAI,CAAC;YACvB,UAAU,CAAC,QAAQ,EAAE,IAAI,EAAE,UAAU,EAAE,SAAS,IAAI,CAAC,CAAC,CAAC,CAAC;IAC5D,MAAM,SAAS,GAAG,cAAc,CAAC,IAAI,CAAC,KAAK,OAAO,CAAC,MAAM,CAAC;IAC1D,MAAM,UAAU,GACd,YAAY,IAAI,SAAS;QACvB,CAAC,CAAC,CAAC,aAAa;aACX,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,gBAAgB,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;aAChD,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC;QAC7C,CAAC,CAAC,EAAE,CAAC;IACT,OAAO;QACL,YAAY;QACZ,UAAU;QACV,QAAQ,EAAE,eAAe,CAAC,YAAY,CAAC,IAAI,CAAC,YAAY;KACzD,CAAC;AACJ,CAAC"}
|