@kensio/pinyinjs 1.7.7 → 1.7.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +20 -4
- package/data/core.entries +2259 -2259
- package/data/core.freq +0 -0
- package/data/full.counts +0 -0
- package/data/full.entries +2289 -2289
- package/data/full.freq +0 -0
- package/data/manifest.json +13 -6
- package/data/standard.entries +2257 -2257
- package/data/standard.freq +0 -0
- package/dist/cli/flag-lists.d.ts +2 -1
- package/dist/cli/flag-lists.d.ts.map +1 -1
- package/dist/cli/flag-lists.js.map +1 -1
- package/dist/cli/format-options.d.ts +5 -1
- package/dist/cli/format-options.d.ts.map +1 -1
- package/dist/cli/format-options.js +9 -2
- package/dist/cli/format-options.js.map +1 -1
- package/dist/cli/html-commands.d.ts.map +1 -1
- package/dist/cli/html-commands.js +15 -3
- package/dist/cli/html-commands.js.map +1 -1
- package/dist/cli/systems.d.ts +15 -43
- package/dist/cli/systems.d.ts.map +1 -1
- package/dist/cli/systems.js +9 -12
- package/dist/cli/systems.js.map +1 -1
- package/dist/decode/guo-rule.d.ts +64 -0
- package/dist/decode/guo-rule.d.ts.map +1 -0
- package/dist/decode/guo-rule.js +138 -0
- package/dist/decode/guo-rule.js.map +1 -0
- package/dist/decode/reading-rules.d.ts +1 -0
- package/dist/decode/reading-rules.d.ts.map +1 -1
- package/dist/decode/reading-rules.js +3 -0
- package/dist/decode/reading-rules.js.map +1 -1
- package/dist/dictionary/artifact-claims.d.ts +8 -0
- package/dist/dictionary/artifact-claims.d.ts.map +1 -1
- package/dist/dictionary/artifact-claims.js +11 -1
- package/dist/dictionary/artifact-claims.js.map +1 -1
- package/dist/dictionary/artifact.d.ts.map +1 -1
- package/dist/dictionary/artifact.js +3 -5
- package/dist/dictionary/artifact.js.map +1 -1
- package/dist/dictionary/frequency-table.d.ts +46 -0
- package/dist/dictionary/frequency-table.d.ts.map +1 -1
- package/dist/dictionary/frequency-table.js +37 -8
- package/dist/dictionary/frequency-table.js.map +1 -1
- package/dist/dictionary/merge-types.d.ts +12 -0
- package/dist/dictionary/merge-types.d.ts.map +1 -1
- package/dist/dictionary/merge-types.js.map +1 -1
- package/dist/dictionary/merge-word.d.ts.map +1 -1
- package/dist/dictionary/merge-word.js +2 -2
- package/dist/dictionary/merge-word.js.map +1 -1
- package/dist/dictionary/merge.d.ts.map +1 -1
- package/dist/dictionary/merge.js +11 -1
- package/dist/dictionary/merge.js.map +1 -1
- package/dist/dictionary/name-mass.d.ts +42 -0
- package/dist/dictionary/name-mass.d.ts.map +1 -0
- package/dist/dictionary/name-mass.js +66 -0
- package/dist/dictionary/name-mass.js.map +1 -0
- package/dist/dictionary/proper-noun.d.ts +8 -1
- package/dist/dictionary/proper-noun.d.ts.map +1 -1
- package/dist/dictionary/proper-noun.js +13 -3
- package/dist/dictionary/proper-noun.js.map +1 -1
- package/dist/dictionary/script-pairings.d.ts +12 -0
- package/dist/dictionary/script-pairings.d.ts.map +1 -1
- package/dist/dictionary/script-pairings.js +20 -0
- package/dist/dictionary/script-pairings.js.map +1 -1
- package/dist/dictionary/source.d.ts +22 -0
- package/dist/dictionary/source.d.ts.map +1 -1
- package/dist/dictionary/source.js +24 -0
- package/dist/dictionary/source.js.map +1 -1
- package/dist/dictionary/traditional-carry.d.ts +23 -0
- package/dist/dictionary/traditional-carry.d.ts.map +1 -0
- package/dist/dictionary/traditional-carry.js +201 -0
- package/dist/dictionary/traditional-carry.js.map +1 -0
- package/dist/dictionary/word-counts.d.ts +74 -0
- package/dist/dictionary/word-counts.d.ts.map +1 -0
- package/dist/dictionary/word-counts.js +142 -0
- package/dist/dictionary/word-counts.js.map +1 -0
- package/dist/format/annotate.d.ts.map +1 -1
- package/dist/format/annotate.js +3 -3
- package/dist/format/annotate.js.map +1 -1
- package/dist/format/html.d.ts +41 -6
- package/dist/format/html.d.ts.map +1 -1
- package/dist/format/html.js +34 -83
- package/dist/format/html.js.map +1 -1
- package/dist/format/language-tag.d.ts +57 -0
- package/dist/format/language-tag.d.ts.map +1 -0
- package/dist/format/language-tag.js +76 -0
- package/dist/format/language-tag.js.map +1 -0
- package/dist/format/syllable-span.d.ts +10 -0
- package/dist/format/syllable-span.d.ts.map +1 -0
- package/dist/format/syllable-span.js +87 -0
- package/dist/format/syllable-span.js.map +1 -0
- package/dist/format/transcribed.d.ts +36 -0
- package/dist/format/transcribed.d.ts.map +1 -0
- package/dist/format/transcribed.js +38 -0
- package/dist/format/transcribed.js.map +1 -0
- package/dist/format/transcription.d.ts.map +1 -1
- package/dist/format/transcription.js +7 -19
- package/dist/format/transcription.js.map +1 -1
- package/dist/index.d.ts +5 -2
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +4 -2
- package/dist/index.js.map +1 -1
- package/dist/transcription/systems.d.ts +82 -0
- package/dist/transcription/systems.d.ts.map +1 -0
- package/dist/transcription/systems.js +64 -0
- package/dist/transcription/systems.js.map +1 -0
- package/package.json +1 -1
- package/dist/cli/system-table.d.ts +0 -12
- package/dist/cli/system-table.d.ts.map +0 -1
- package/dist/cli/system-table.js +0 -48
- package/dist/cli/system-table.js.map +0 -1
|
@@ -2,6 +2,7 @@ import { type ScriptTables } from "../script/conversion.js";
|
|
|
2
2
|
import type { DictionaryArtifact } from "./artifact.js";
|
|
3
3
|
import { Dictionary } from "./dictionary.js";
|
|
4
4
|
import type { Tier } from "./tiers.js";
|
|
5
|
+
import { WordCounts } from "./word-counts.js";
|
|
5
6
|
/**
|
|
6
7
|
* Where a dictionary's files are read from.
|
|
7
8
|
*
|
|
@@ -43,6 +44,27 @@ export declare function loadArtifact(source: DictionarySource, tier: Tier): Prom
|
|
|
43
44
|
* Load a tier and wrap it in a {@link Dictionary}.
|
|
44
45
|
*/
|
|
45
46
|
export declare function loadDictionary(source: DictionarySource, tier: Tier): Promise<Dictionary>;
|
|
47
|
+
/**
|
|
48
|
+
* The file the raw corpus counts live in.
|
|
49
|
+
*
|
|
50
|
+
* `full` alone, and outside the tiers rather than one per tier: a caller
|
|
51
|
+
* ranking words wants the whole vocabulary ordered, and a ranking over part of
|
|
52
|
+
* it answers a different question.
|
|
53
|
+
*/
|
|
54
|
+
export declare const COUNTS_FILE = "full.counts";
|
|
55
|
+
/**
|
|
56
|
+
* Load the raw corpus counts for the `full` tier.
|
|
57
|
+
*
|
|
58
|
+
* Separate from the dictionary for the same reason {@link loadScriptTables} is.
|
|
59
|
+
* Nothing on the decoding path reads counts, and the quantised frequencies the
|
|
60
|
+
* decoder does read already ship inside every tier. This file is 243 KB brotli
|
|
61
|
+
* against `full.freq`'s 148 KB, and only a caller that asks for it pays.
|
|
62
|
+
*
|
|
63
|
+
* The counts are positional over the `full` tier's keys, so pair them with a
|
|
64
|
+
* `full` {@link Dictionary} and nothing else. {@link WordCounts.size} and
|
|
65
|
+
* {@link Dictionary.size} agree where the two match.
|
|
66
|
+
*/
|
|
67
|
+
export declare function loadWordCounts(source: DictionarySource): Promise<WordCounts>;
|
|
46
68
|
/**
|
|
47
69
|
* The file the script conversion tables live in.
|
|
48
70
|
*
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"source.d.ts","sourceRoot":"","sources":["../../src/dictionary/source.ts"],"names":[],"mappings":"AAAA,OAAO,EAAoB,KAAK,YAAY,EAAE,MAAM,yBAAyB,CAAC;AAC9E,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,eAAe,CAAC;AACxD,OAAO,EAAE,UAAU,EAAE,MAAM,iBAAiB,CAAC;AAC7C,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;
|
|
1
|
+
{"version":3,"file":"source.d.ts","sourceRoot":"","sources":["../../src/dictionary/source.ts"],"names":[],"mappings":"AAAA,OAAO,EAAoB,KAAK,YAAY,EAAE,MAAM,yBAAyB,CAAC;AAC9E,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,eAAe,CAAC;AACxD,OAAO,EAAE,UAAU,EAAE,MAAM,iBAAiB,CAAC;AAC7C,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AACvC,OAAO,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAC;AAE9C;;;;;;;GAOG;AACH,MAAM,WAAW,gBAAgB;IAC/B,0CAA0C;IAC1C,QAAQ,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,OAAO,CAAC,MAAM,CAAC,CAAC;IACjD,2CAA2C;IAC3C,QAAQ,CAAC,KAAK,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,OAAO,CAAC,UAAU,CAAC,CAAC;CACvD;AAED;;GAEG;AACH,wBAAgB,SAAS,CAAC,IAAI,EAAE,IAAI,GAAG;IACrC,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;CAC9B,CAMA;AAED;;;;;;GAMG;AACH,wBAAgB,WAAW,CAAC,OAAO,EAAE,MAAM,GAAG,gBAAgB,CAqB7D;AAED;;;;;GAKG;AACH,wBAAsB,YAAY,CAChC,MAAM,EAAE,gBAAgB,EACxB,IAAI,EAAE,IAAI,GACT,OAAO,CAAC,kBAAkB,CAAC,CAQ7B;AAED;;GAEG;AACH,wBAAsB,cAAc,CAClC,MAAM,EAAE,gBAAgB,EACxB,IAAI,EAAE,IAAI,GACT,OAAO,CAAC,UAAU,CAAC,CAErB;AAED;;;;;;GAMG;AACH,eAAO,MAAM,WAAW,gBAAgB,CAAC;AAEzC;;;;;;;;;;;GAWG;AACH,wBAAsB,cAAc,CAClC,MAAM,EAAE,gBAAgB,GACvB,OAAO,CAAC,UAAU,CAAC,CAErB;AAED;;;;;;GAMG;AACH,eAAO,MAAM,WAAW,eAAe,CAAC;AAExC;;;;;;;GAOG;AACH,wBAAsB,gBAAgB,CACpC,MAAM,EAAE,gBAAgB,GACvB,OAAO,CAAC,YAAY,CAAC,CAEvB"}
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { readScriptTables } from "../script/conversion.js";
|
|
2
2
|
import { Dictionary } from "./dictionary.js";
|
|
3
|
+
import { WordCounts } from "./word-counts.js";
|
|
3
4
|
/**
|
|
4
5
|
* The three files a tier is made of.
|
|
5
6
|
*/
|
|
@@ -58,6 +59,29 @@ export async function loadArtifact(source, tier) {
|
|
|
58
59
|
export async function loadDictionary(source, tier) {
|
|
59
60
|
return Dictionary.from(await loadArtifact(source, tier));
|
|
60
61
|
}
|
|
62
|
+
/**
|
|
63
|
+
* The file the raw corpus counts live in.
|
|
64
|
+
*
|
|
65
|
+
* `full` alone, and outside the tiers rather than one per tier: a caller
|
|
66
|
+
* ranking words wants the whole vocabulary ordered, and a ranking over part of
|
|
67
|
+
* it answers a different question.
|
|
68
|
+
*/
|
|
69
|
+
export const COUNTS_FILE = "full.counts";
|
|
70
|
+
/**
|
|
71
|
+
* Load the raw corpus counts for the `full` tier.
|
|
72
|
+
*
|
|
73
|
+
* Separate from the dictionary for the same reason {@link loadScriptTables} is.
|
|
74
|
+
* Nothing on the decoding path reads counts, and the quantised frequencies the
|
|
75
|
+
* decoder does read already ship inside every tier. This file is 243 KB brotli
|
|
76
|
+
* against `full.freq`'s 148 KB, and only a caller that asks for it pays.
|
|
77
|
+
*
|
|
78
|
+
* The counts are positional over the `full` tier's keys, so pair them with a
|
|
79
|
+
* `full` {@link Dictionary} and nothing else. {@link WordCounts.size} and
|
|
80
|
+
* {@link Dictionary.size} agree where the two match.
|
|
81
|
+
*/
|
|
82
|
+
export async function loadWordCounts(source) {
|
|
83
|
+
return WordCounts.from(await source.bytes(COUNTS_FILE));
|
|
84
|
+
}
|
|
61
85
|
/**
|
|
62
86
|
* The file the script conversion tables live in.
|
|
63
87
|
*
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"source.js","sourceRoot":"","sources":["../../src/dictionary/source.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,gBAAgB,EAAqB,MAAM,yBAAyB,CAAC;AAE9E,OAAO,EAAE,UAAU,EAAE,MAAM,iBAAiB,CAAC;
|
|
1
|
+
{"version":3,"file":"source.js","sourceRoot":"","sources":["../../src/dictionary/source.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,gBAAgB,EAAqB,MAAM,yBAAyB,CAAC;AAE9E,OAAO,EAAE,UAAU,EAAE,MAAM,iBAAiB,CAAC;AAE7C,OAAO,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAC;AAiB9C;;GAEG;AACH,MAAM,UAAU,SAAS,CAAC,IAAU;IAKlC,OAAO;QACL,IAAI,EAAE,GAAG,IAAI,OAAO;QACpB,OAAO,EAAE,GAAG,IAAI,UAAU;QAC1B,WAAW,EAAE,GAAG,IAAI,OAAO;KAC5B,CAAC;AACJ,CAAC;AAED;;;;;;GAMG;AACH,MAAM,UAAU,WAAW,CAAC,OAAe;IACzC,MAAM,EAAE,GAAG,CAAC,IAAY,EAAU,EAAE,CAAC,GAAG,OAAO,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,IAAI,IAAI,EAAE,CAAC;IAC9E,MAAM,GAAG,GAAG,KAAK,EAAE,IAAY,EAAqB,EAAE;QACpD,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,EAAE,CAAC,IAAI,CAAC,CAAC,CAAC;QACvC,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;YACjB,MAAM,IAAI,KAAK,CACb,YAAY,EAAE,CAAC,IAAI,CAAC,YAAY,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,QAAQ,CAAC,UAAU,EAAE,CACjF,CAAC;QACJ,CAAC;QACD,OAAO,QAAQ,CAAC;IAClB,CAAC,CAAC;IACF,OAAO;QACL,IAAI,EAAE,KAAK,EAAE,IAAY,EAAmB,EAAE;YAC5C,MAAM,QAAQ,GAAG,MAAM,GAAG,CAAC,IAAI,CAAC,CAAC;YACjC,OAAO,QAAQ,CAAC,IAAI,EAAE,CAAC;QACzB,CAAC;QACD,KAAK,EAAE,KAAK,EAAE,IAAY,EAAuB,EAAE;YACjD,MAAM,QAAQ,GAAG,MAAM,GAAG,CAAC,IAAI,CAAC,CAAC;YACjC,OAAO,IAAI,UAAU,CAAC,MAAM,QAAQ,CAAC,WAAW,EAAE,CAAC,CAAC;QACtD,CAAC;KACF,CAAC;AACJ,CAAC;AAED;;;;;GAKG;AACH,MAAM,CAAC,KAAK,UAAU,YAAY,CAChC,MAAwB,EACxB,IAAU;IAEV,MAAM,KAAK,GAAG,SAAS,CAAC,IAAI,CAAC,CAAC;IAC9B,MAAM,CAAC,IAAI,EAAE,OAAO,EAAE,WAAW,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC;QACrD,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC;QACvB,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC;QAC1B,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,WAAW,CAAC;KAChC,CAAC,CAAC;IACH,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,WAAW,EAAE,CAAC;AACxC,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,cAAc,CAClC,MAAwB,EACxB,IAAU;IAEV,OAAO,UAAU,CAAC,IAAI,CAAC,MAAM,YAAY,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC,CAAC;AAC3D,CAAC;AAED;;;;;;GAMG;AACH,MAAM,CAAC,MAAM,WAAW,GAAG,aAAa,CAAC;AAEzC;;;;;;;;;;;GAWG;AACH,MAAM,CAAC,KAAK,UAAU,cAAc,CAClC,MAAwB;IAExB,OAAO,UAAU,CAAC,IAAI,CAAC,MAAM,MAAM,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC,CAAC;AAC1D,CAAC;AAED;;;;;;GAMG;AACH,MAAM,CAAC,MAAM,WAAW,GAAG,YAAY,CAAC;AAExC;;;;;;;GAOG;AACH,MAAM,CAAC,KAAK,UAAU,gBAAgB,CACpC,MAAwB;IAExB,OAAO,gBAAgB,CAAC,MAAM,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC,CAAC;AAC1D,CAAC"}
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
import type { DictionaryEntry } from "./entry.js";
|
|
2
|
+
/**
|
|
3
|
+
* Whether a tag says anything about the word.
|
|
4
|
+
*/
|
|
5
|
+
export declare function isTagged(partOfSpeech: string): boolean;
|
|
6
|
+
/**
|
|
7
|
+
* The entries after the carry, and how many took each field.
|
|
8
|
+
*/
|
|
9
|
+
export interface CarriedEntries {
|
|
10
|
+
readonly entries: readonly DictionaryEntry[];
|
|
11
|
+
readonly carriedTags: number;
|
|
12
|
+
readonly carriedCounts: number;
|
|
13
|
+
readonly carriedCapitals: number;
|
|
14
|
+
}
|
|
15
|
+
/**
|
|
16
|
+
* Give every 繁體 headword the tag and the count its 简体 word carries.
|
|
17
|
+
*
|
|
18
|
+
* Neither overwrites something the source stated. A spelling jieba classified
|
|
19
|
+
* keeps its own tag, and a spelling jieba counted more often than its 简体 word
|
|
20
|
+
* keeps its own count.
|
|
21
|
+
*/
|
|
22
|
+
export declare function carryToTraditional(entries: readonly DictionaryEntry[]): CarriedEntries;
|
|
23
|
+
//# sourceMappingURL=traditional-carry.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"traditional-carry.d.ts","sourceRoot":"","sources":["../../src/dictionary/traditional-carry.ts"],"names":[],"mappings":"AA6BA,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,YAAY,CAAC;AAkBlD;;GAEG;AACH,wBAAgB,QAAQ,CAAC,YAAY,EAAE,MAAM,GAAG,OAAO,CAEtD;AAED;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B,QAAQ,CAAC,OAAO,EAAE,SAAS,eAAe,EAAE,CAAC;IAC7C,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,aAAa,EAAE,MAAM,CAAC;IAC/B,QAAQ,CAAC,eAAe,EAAE,MAAM,CAAC;CAClC;AAqID;;;;;;GAMG;AACH,wBAAgB,kBAAkB,CAChC,OAAO,EAAE,SAAS,eAAe,EAAE,GAClC,cAAc,CA0ChB"}
|
|
@@ -0,0 +1,201 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* What a 繁體 spelling takes from the 简体 word it pairs with.
|
|
3
|
+
*
|
|
4
|
+
* jieba's dictionary was counted over a 简体 corpus, and all three of the fields
|
|
5
|
+
* it settles arrive on one script and not the other. 听 is tagged `v` and
|
|
6
|
+
* counted 20,435 times. 聽 is tagged nothing, counted nowhere, and 麥 is not a
|
|
7
|
+
* name where 麦 is one. Every rule that asks what the word beside it is decides
|
|
8
|
+
* on the tag, every path the decoder weighs is priced on the count, and the
|
|
9
|
+
* capital comes straight off the proper-noun bit, so 繁體 text was read by
|
|
10
|
+
* rules that could not see it, priced by a model that had never met it and
|
|
11
|
+
* capitalised by a different answer from the one 简体 got.
|
|
12
|
+
*
|
|
13
|
+
* All three travel over the pairing the entries already hold. An entry names
|
|
14
|
+
* the 繁體 spelling of its own word. The entry keyed on that spelling is
|
|
15
|
+
* therefore the same word written the other way, and how a word is classed, how
|
|
16
|
+
* often it is met and whether it is a name are facts about the word rather than
|
|
17
|
+
* about the spelling. `corpus-mass.ts` makes the same move for the polyphone
|
|
18
|
+
* priors, where a vote cast for a 简体 word is cast again for whichever 繁體
|
|
19
|
+
* form the reading picks.
|
|
20
|
+
*
|
|
21
|
+
* A 繁體 spelling that is not a headword of its own needs none of this, because
|
|
22
|
+
* `artifact-claims.ts` lets its 简体 entry claim the key outright and the key
|
|
23
|
+
* carries that entry's fields. 銀行 is claimed by 银行 and always priced as it.
|
|
24
|
+
* What is left is the characters Unihan gives an entry to, which is why only
|
|
25
|
+
* single characters are paired here.
|
|
26
|
+
*/
|
|
27
|
+
import { convertCharacter } from "../script/conversion.js";
|
|
28
|
+
import { isSingleCharacter } from "../script/characters.js";
|
|
29
|
+
import { traditionalCharacterTable } from "./script-pairings.js";
|
|
30
|
+
/**
|
|
31
|
+
* jieba's tag for a character it has counted and not classified.
|
|
32
|
+
*
|
|
33
|
+
* Written on 5,666 entries and on nothing longer than one character, 简体 and
|
|
34
|
+
* 繁體 alike: 且, 丢, 乎, 井, 今 and 仍 carry it next to 來, 個 and 這. It says
|
|
35
|
+
* that the character was seen rather than what it is. Read here as no tag at
|
|
36
|
+
* all, and left in the artifact, because whether jieba wrote `zg` or wrote
|
|
37
|
+
* nothing is a fact about the source and only this decides what to make of it.
|
|
38
|
+
*
|
|
39
|
+
* Nothing longer than one character carries it. That leaves the rules forbidding
|
|
40
|
+
* an **untagged key** alone, since each of those asks about a word of two
|
|
41
|
+
* characters or more and no such key is tagged `zg`.
|
|
42
|
+
*/
|
|
43
|
+
const UNCATEGORISED = "zg";
|
|
44
|
+
/**
|
|
45
|
+
* Whether a tag says anything about the word.
|
|
46
|
+
*/
|
|
47
|
+
export function isTagged(partOfSpeech) {
|
|
48
|
+
return partOfSpeech !== "" && partOfSpeech !== UNCATEGORISED;
|
|
49
|
+
}
|
|
50
|
+
/**
|
|
51
|
+
* The 繁體 character a 简体 one is written as, or undefined where it is neither.
|
|
52
|
+
*
|
|
53
|
+
* Read off the aggregate table rather than off {@link DictionaryEntry.hant},
|
|
54
|
+
* and that is the whole of what keeps this honest. An entry's own 繁體 form is
|
|
55
|
+
* whichever CC-CEDICT sense matched its reading, and for a single character
|
|
56
|
+
* that can be an oddity: CC-CEDICT holds `旹 时` as an old variant of 時, so
|
|
57
|
+
* 时's entry can name 旹 while every word 时 appears in says 時. Lending 时's
|
|
58
|
+
* 103,735 occurrences to 旹 made a character nobody writes the third commonest
|
|
59
|
+
* way to write `shí`. The table counts words, so 時 wins it thousands to one.
|
|
60
|
+
*/
|
|
61
|
+
function traditionalOf(table, entry) {
|
|
62
|
+
if (!isSingleCharacter(entry.hans)) {
|
|
63
|
+
return undefined;
|
|
64
|
+
}
|
|
65
|
+
const written = convertCharacter(table, entry.hans, entry.readings.cn[0]);
|
|
66
|
+
return written === entry.hans ? undefined : written;
|
|
67
|
+
}
|
|
68
|
+
/**
|
|
69
|
+
* The 简体 entry each 繁體 character should take a field from.
|
|
70
|
+
*
|
|
71
|
+
* Where two 简体 characters are written the same way in 繁體 the commoner of
|
|
72
|
+
* them lends. Both fields are being used to guess at what a reader meant, and
|
|
73
|
+
* the commoner character is the likelier thing to have been meant. `hasField`
|
|
74
|
+
* is asked first, so that a commoner character with nothing to lend does not
|
|
75
|
+
* shut out a rarer one that has something.
|
|
76
|
+
*/
|
|
77
|
+
function lendersByTraditional(entries, table, hasField) {
|
|
78
|
+
const lenders = new Map();
|
|
79
|
+
for (const entry of entries) {
|
|
80
|
+
const written = hasField(entry) ? traditionalOf(table, entry) : undefined;
|
|
81
|
+
if (written === undefined) {
|
|
82
|
+
continue;
|
|
83
|
+
}
|
|
84
|
+
const held = lenders.get(written);
|
|
85
|
+
if (held === undefined || held.frequency < entry.frequency) {
|
|
86
|
+
lenders.set(written, entry);
|
|
87
|
+
}
|
|
88
|
+
}
|
|
89
|
+
return lenders;
|
|
90
|
+
}
|
|
91
|
+
/**
|
|
92
|
+
* Whether an entry has a tag worth lending.
|
|
93
|
+
*/
|
|
94
|
+
function canLendTag(entry) {
|
|
95
|
+
return isTagged(entry.partOfSpeech);
|
|
96
|
+
}
|
|
97
|
+
/**
|
|
98
|
+
* The tag a spelling ends up with, and whether it took one.
|
|
99
|
+
*
|
|
100
|
+
* `nr`, `ns`, `nt` and `nz` are lent along with the rest now that
|
|
101
|
+
* {@link properFor} carries the bit they go with. Lending the tag without it
|
|
102
|
+
* would have left a spelling claiming a place name the same entry denies is a
|
|
103
|
+
* proper noun, which is why they were held back until the bit travelled.
|
|
104
|
+
*/
|
|
105
|
+
function tagFor(entry, lender) {
|
|
106
|
+
if (isTagged(entry.partOfSpeech) || lender === undefined) {
|
|
107
|
+
return undefined;
|
|
108
|
+
}
|
|
109
|
+
return lender.partOfSpeech;
|
|
110
|
+
}
|
|
111
|
+
/**
|
|
112
|
+
* The count a spelling ends up with, and whether it took one.
|
|
113
|
+
*
|
|
114
|
+
* The larger of the two, rather than the lender's outright. A 繁體 spelling
|
|
115
|
+
* that jieba counted at all was counted incidentally in a corpus written the
|
|
116
|
+
* other way, and the word occurs at least as often as the commoner spelling of
|
|
117
|
+
* it was seen. Taking the smaller would price 過 by the handful of 繁體 lines
|
|
118
|
+
* jieba's corpus happens to contain.
|
|
119
|
+
*
|
|
120
|
+
* Only the form the reading picks takes it. 发 read `fā` is written 發, so 發
|
|
121
|
+
* takes 发's count and 髮 does not, which is the same line the conversion draws
|
|
122
|
+
* when it writes 出发 as 出發 and 头发 as 頭髮.
|
|
123
|
+
*
|
|
124
|
+
* **The count is carried whole and not shared out.** A share would say that
|
|
125
|
+
* 繁體 text is some fraction as common as 简体 text, and nothing measured here
|
|
126
|
+
* says what that fraction is. It would also be invisible: the buckets the
|
|
127
|
+
* decoder compares are log-spaced, one of them is a factor of 2.49, so halving
|
|
128
|
+
* a count moves it 0.76 of a bucket and a tenth of it moves it 2.5. Only a
|
|
129
|
+
* share of an order of magnitude would change a decode, and inventing one that
|
|
130
|
+
* large to fix a decode that is currently wrong would be trading one guess for
|
|
131
|
+
* another.
|
|
132
|
+
*/
|
|
133
|
+
function countFor(entry, lender) {
|
|
134
|
+
if (lender === undefined || lender.frequency <= entry.frequency) {
|
|
135
|
+
return undefined;
|
|
136
|
+
}
|
|
137
|
+
return lender.frequency;
|
|
138
|
+
}
|
|
139
|
+
/**
|
|
140
|
+
* Whether a spelling is a name, and whether that is a change.
|
|
141
|
+
*
|
|
142
|
+
* The bit is what survived `properNounOf`, so it is the answer both sources
|
|
143
|
+
* reached together rather than jieba's proposal, and it is taken whichever way
|
|
144
|
+
* it points. Demoting matters as much as promoting. 后 is a locative and 後
|
|
145
|
+
* arrived tagged `nr` with nothing under its own spelling to challenge it, so
|
|
146
|
+
* 退休後 came out `tuìxiū Hòu` where 退休后 came out `tuìxiū hòu`.
|
|
147
|
+
*
|
|
148
|
+
* This carries the 简体 answer whether or not that answer is right. jieba calls
|
|
149
|
+
* 连 a surname and 連 now agrees, where before the two differed and one of them
|
|
150
|
+
* happened to be correct. The two scripts saying one thing is what this is for.
|
|
151
|
+
* How good that one thing is for a bare character is #159.
|
|
152
|
+
*/
|
|
153
|
+
function properFor(entry, lender) {
|
|
154
|
+
if (lender === undefined || lender.isProperNoun === entry.isProperNoun) {
|
|
155
|
+
return undefined;
|
|
156
|
+
}
|
|
157
|
+
return lender.isProperNoun;
|
|
158
|
+
}
|
|
159
|
+
/**
|
|
160
|
+
* Give every 繁體 headword the tag and the count its 简体 word carries.
|
|
161
|
+
*
|
|
162
|
+
* Neither overwrites something the source stated. A spelling jieba classified
|
|
163
|
+
* keeps its own tag, and a spelling jieba counted more often than its 简体 word
|
|
164
|
+
* keeps its own count.
|
|
165
|
+
*/
|
|
166
|
+
export function carryToTraditional(entries) {
|
|
167
|
+
const table = traditionalCharacterTable(entries);
|
|
168
|
+
const tagLenders = lendersByTraditional(entries, table, canLendTag);
|
|
169
|
+
const countLenders = lendersByTraditional(entries, table, (entry) => entry.frequency > 0);
|
|
170
|
+
let carriedTags = 0;
|
|
171
|
+
let carriedCounts = 0;
|
|
172
|
+
let carriedCapitals = 0;
|
|
173
|
+
const carried = entries.map((entry) => {
|
|
174
|
+
const tagLender = tagLenders.get(entry.hans);
|
|
175
|
+
const partOfSpeech = tagFor(entry, tagLender);
|
|
176
|
+
const frequency = countFor(entry, countLenders.get(entry.hans));
|
|
177
|
+
const isProperNoun = properFor(entry, tagLender);
|
|
178
|
+
if (partOfSpeech === undefined &&
|
|
179
|
+
frequency === undefined &&
|
|
180
|
+
isProperNoun === undefined) {
|
|
181
|
+
return entry;
|
|
182
|
+
}
|
|
183
|
+
if (partOfSpeech !== undefined) {
|
|
184
|
+
carriedTags += 1;
|
|
185
|
+
}
|
|
186
|
+
if (frequency !== undefined) {
|
|
187
|
+
carriedCounts += 1;
|
|
188
|
+
}
|
|
189
|
+
if (isProperNoun !== undefined) {
|
|
190
|
+
carriedCapitals += 1;
|
|
191
|
+
}
|
|
192
|
+
return {
|
|
193
|
+
...entry,
|
|
194
|
+
...(partOfSpeech !== undefined && { partOfSpeech }),
|
|
195
|
+
...(frequency !== undefined && { frequency }),
|
|
196
|
+
...(isProperNoun !== undefined && { isProperNoun }),
|
|
197
|
+
};
|
|
198
|
+
});
|
|
199
|
+
return { entries: carried, carriedTags, carriedCounts, carriedCapitals };
|
|
200
|
+
}
|
|
201
|
+
//# sourceMappingURL=traditional-carry.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"traditional-carry.js","sourceRoot":"","sources":["../../src/dictionary/traditional-carry.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;GAyBG;AACH,OAAO,EAAE,gBAAgB,EAAE,MAAM,yBAAyB,CAAC;AAE3D,OAAO,EAAE,iBAAiB,EAAE,MAAM,yBAAyB,CAAC;AAE5D,OAAO,EAAE,yBAAyB,EAAE,MAAM,sBAAsB,CAAC;AAEjE;;;;;;;;;;;;GAYG;AACH,MAAM,aAAa,GAAG,IAAI,CAAC;AAE3B;;GAEG;AACH,MAAM,UAAU,QAAQ,CAAC,YAAoB;IAC3C,OAAO,YAAY,KAAK,EAAE,IAAI,YAAY,KAAK,aAAa,CAAC;AAC/D,CAAC;AAYD;;;;;;;;;;GAUG;AACH,SAAS,aAAa,CACpB,KAA+C,EAC/C,KAAsB;IAEtB,IAAI,CAAC,iBAAiB,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC;QACnC,OAAO,SAAS,CAAC;IACnB,CAAC;IACD,MAAM,OAAO,GAAG,gBAAgB,CAAC,KAAK,EAAE,KAAK,CAAC,IAAI,EAAE,KAAK,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;IAC1E,OAAO,OAAO,KAAK,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,OAAO,CAAC;AACtD,CAAC;AAED;;;;;;;;GAQG;AACH,SAAS,oBAAoB,CAC3B,OAAmC,EACnC,KAA+C,EAC/C,QAA6C;IAE7C,MAAM,OAAO,GAAG,IAAI,GAAG,EAA2B,CAAC;IACnD,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;QAC5B,MAAM,OAAO,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,aAAa,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;QAC1E,IAAI,OAAO,KAAK,SAAS,EAAE,CAAC;YAC1B,SAAS;QACX,CAAC;QACD,MAAM,IAAI,GAAG,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;QAClC,IAAI,IAAI,KAAK,SAAS,IAAI,IAAI,CAAC,SAAS,GAAG,KAAK,CAAC,SAAS,EAAE,CAAC;YAC3D,OAAO,CAAC,GAAG,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC;QAC9B,CAAC;IACH,CAAC;IACD,OAAO,OAAO,CAAC;AACjB,CAAC;AAED;;GAEG;AACH,SAAS,UAAU,CAAC,KAAsB;IACxC,OAAO,QAAQ,CAAC,KAAK,CAAC,YAAY,CAAC,CAAC;AACtC,CAAC;AAED;;;;;;;GAOG;AACH,SAAS,MAAM,CACb,KAAsB,EACtB,MAAmC;IAEnC,IAAI,QAAQ,CAAC,KAAK,CAAC,YAAY,CAAC,IAAI,MAAM,KAAK,SAAS,EAAE,CAAC;QACzD,OAAO,SAAS,CAAC;IACnB,CAAC;IACD,OAAO,MAAM,CAAC,YAAY,CAAC;AAC7B,CAAC;AAED;;;;;;;;;;;;;;;;;;;;;GAqBG;AACH,SAAS,QAAQ,CACf,KAAsB,EACtB,MAAmC;IAEnC,IAAI,MAAM,KAAK,SAAS,IAAI,MAAM,CAAC,SAAS,IAAI,KAAK,CAAC,SAAS,EAAE,CAAC;QAChE,OAAO,SAAS,CAAC;IACnB,CAAC;IACD,OAAO,MAAM,CAAC,SAAS,CAAC;AAC1B,CAAC;AAED;;;;;;;;;;;;;GAaG;AACH,SAAS,SAAS,CAChB,KAAsB,EACtB,MAAmC;IAEnC,IAAI,MAAM,KAAK,SAAS,IAAI,MAAM,CAAC,YAAY,KAAK,KAAK,CAAC,YAAY,EAAE,CAAC;QACvE,OAAO,SAAS,CAAC;IACnB,CAAC;IACD,OAAO,MAAM,CAAC,YAAY,CAAC;AAC7B,CAAC;AAED;;;;;;GAMG;AACH,MAAM,UAAU,kBAAkB,CAChC,OAAmC;IAEnC,MAAM,KAAK,GAAG,yBAAyB,CAAC,OAAO,CAAC,CAAC;IACjD,MAAM,UAAU,GAAG,oBAAoB,CAAC,OAAO,EAAE,KAAK,EAAE,UAAU,CAAC,CAAC;IACpE,MAAM,YAAY,GAAG,oBAAoB,CACvC,OAAO,EACP,KAAK,EACL,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,SAAS,GAAG,CAAC,CAC/B,CAAC;IACF,IAAI,WAAW,GAAG,CAAC,CAAC;IACpB,IAAI,aAAa,GAAG,CAAC,CAAC;IACtB,IAAI,eAAe,GAAG,CAAC,CAAC;IAExB,MAAM,OAAO,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE;QACpC,MAAM,SAAS,GAAG,UAAU,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QAC7C,MAAM,YAAY,GAAG,MAAM,CAAC,KAAK,EAAE,SAAS,CAAC,CAAC;QAC9C,MAAM,SAAS,GAAG,QAAQ,CAAC,KAAK,EAAE,YAAY,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC;QAChE,MAAM,YAAY,GAAG,SAAS,CAAC,KAAK,EAAE,SAAS,CAAC,CAAC;QACjD,IACE,YAAY,KAAK,SAAS;YAC1B,SAAS,KAAK,SAAS;YACvB,YAAY,KAAK,SAAS,EAC1B,CAAC;YACD,OAAO,KAAK,CAAC;QACf,CAAC;QACD,IAAI,YAAY,KAAK,SAAS,EAAE,CAAC;YAC/B,WAAW,IAAI,CAAC,CAAC;QACnB,CAAC;QACD,IAAI,SAAS,KAAK,SAAS,EAAE,CAAC;YAC5B,aAAa,IAAI,CAAC,CAAC;QACrB,CAAC;QACD,IAAI,YAAY,KAAK,SAAS,EAAE,CAAC;YAC/B,eAAe,IAAI,CAAC,CAAC;QACvB,CAAC;QACD,OAAO;YACL,GAAG,KAAK;YACR,GAAG,CAAC,YAAY,KAAK,SAAS,IAAI,EAAE,YAAY,EAAE,CAAC;YACnD,GAAG,CAAC,SAAS,KAAK,SAAS,IAAI,EAAE,SAAS,EAAE,CAAC;YAC7C,GAAG,CAAC,YAAY,KAAK,SAAS,IAAI,EAAE,YAAY,EAAE,CAAC;SACpD,CAAC;IACJ,CAAC,CAAC,CAAC;IAEH,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,WAAW,EAAE,aAAa,EAAE,eAAe,EAAE,CAAC;AAC3E,CAAC"}
|
|
@@ -0,0 +1,74 @@
|
|
|
1
|
+
import type { DictionaryEntry } from "./entry.js";
|
|
2
|
+
/**
|
|
3
|
+
* Corpus counts for every key of a tier, one variable-length integer each.
|
|
4
|
+
*
|
|
5
|
+
* Counts are positional in the same sense the frequency table is: entry *n*
|
|
6
|
+
* describes the key at position *n* of the matching
|
|
7
|
+
* {@link import("./key-index.js").KeyIndex}, which is what
|
|
8
|
+
* {@link import("./dictionary.js").Dictionary.wordAt} reads.
|
|
9
|
+
*
|
|
10
|
+
* The encoding is LEB128, measured against the alternatives on the full tier's
|
|
11
|
+
* 723,147 keys. It takes 747 KB, and 243 KB brotli, where a fixed 32-bit array
|
|
12
|
+
* takes 2,825 KB and 299 KB and decimal text takes 1,586 KB and 281 KB. Two
|
|
13
|
+
* thirds of the keys are attested nowhere and most of the rest carry a small
|
|
14
|
+
* count, and a fixed width pays for the maximum on all of them.
|
|
15
|
+
*
|
|
16
|
+
* Unlike the rest of the artifact this *is* parsed on load, into one typed
|
|
17
|
+
* array. Nothing in a browser reads it, so the load cost buys a random access
|
|
18
|
+
* that a varint stream cannot otherwise give.
|
|
19
|
+
*/
|
|
20
|
+
export declare class WordCounts {
|
|
21
|
+
#private;
|
|
22
|
+
/**
|
|
23
|
+
* Encode raw counts, in key order.
|
|
24
|
+
*
|
|
25
|
+
* A count of zero means the word is attested nowhere in the corpus, which is
|
|
26
|
+
* a fact about the word and is stored as one. Fractions and negatives are
|
|
27
|
+
* floored into range rather than rejected, since a source that supplies one
|
|
28
|
+
* has already been through the merge.
|
|
29
|
+
*/
|
|
30
|
+
static build(counts: readonly number[]): WordCounts;
|
|
31
|
+
/**
|
|
32
|
+
* Decode packed bytes read back from an artifact.
|
|
33
|
+
*
|
|
34
|
+
* How many counts there are is a property of the stream rather than an
|
|
35
|
+
* argument, since every count ends at the first byte without a continuation
|
|
36
|
+
* bit. A truncated stream ends mid-count, and the trailing bytes are dropped
|
|
37
|
+
* rather than reported as a value the file never held.
|
|
38
|
+
*/
|
|
39
|
+
static from(packed: Uint8Array): WordCounts;
|
|
40
|
+
/**
|
|
41
|
+
* Hold decoded counts beside the bytes they came from. Use
|
|
42
|
+
* {@link WordCounts.build} or {@link WordCounts.from}.
|
|
43
|
+
*/
|
|
44
|
+
private constructor();
|
|
45
|
+
/**
|
|
46
|
+
* How many keys the counts describe.
|
|
47
|
+
*
|
|
48
|
+
* Worth checking against {@link import("./dictionary.js").Dictionary.size}
|
|
49
|
+
* before pairing the two by position. The counts are built for one tier, and
|
|
50
|
+
* against any other tier every position would name a different word.
|
|
51
|
+
*/
|
|
52
|
+
get size(): number;
|
|
53
|
+
/**
|
|
54
|
+
* Corpus occurrences of the key at a position.
|
|
55
|
+
*
|
|
56
|
+
* Positions outside the table report zero, which is what an unattested word
|
|
57
|
+
* reports, so a caller ranking by count sorts a missing key to the bottom.
|
|
58
|
+
*/
|
|
59
|
+
countOf(at: number): number;
|
|
60
|
+
/**
|
|
61
|
+
* The packed bytes, ready to be written out.
|
|
62
|
+
*/
|
|
63
|
+
serialise(): Uint8Array;
|
|
64
|
+
}
|
|
65
|
+
/**
|
|
66
|
+
* Compile the counts artifact from merged entries.
|
|
67
|
+
*
|
|
68
|
+
* The keys are claimed and ordered the way `artifact.ts` claims and orders
|
|
69
|
+
* them, so position *n* here is position *n* there. Only the `full` tier is
|
|
70
|
+
* written: a caller ranking words wants the whole vocabulary ordered, and a
|
|
71
|
+
* ranking over part of it answers a different question.
|
|
72
|
+
*/
|
|
73
|
+
export declare function buildWordCounts(entries: readonly DictionaryEntry[]): WordCounts;
|
|
74
|
+
//# sourceMappingURL=word-counts.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"word-counts.d.ts","sourceRoot":"","sources":["../../src/dictionary/word-counts.ts"],"names":[],"mappings":"AAgBA,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,YAAY,CAAC;AAelD;;;;;;;;;;;;;;;;;GAiBG;AACH,qBAAa,UAAU;;IACrB;;;;;;;OAOG;IACH,MAAM,CAAC,KAAK,CAAC,MAAM,EAAE,SAAS,MAAM,EAAE,GAAG,UAAU,CAgBlD;IAED;;;;;;;OAOG;IACH,MAAM,CAAC,IAAI,CAAC,MAAM,EAAE,UAAU,GAAG,UAAU,CAkB1C;IAKD;;;OAGG;IACH,OAAO,eAGN;IAED;;;;;;OAMG;IACH,IAAI,IAAI,IAAI,MAAM,CAEjB;IAED;;;;;OAKG;IACH,OAAO,CAAC,EAAE,EAAE,MAAM,GAAG,MAAM,CAE1B;IAED;;OAEG;IACH,SAAS,IAAI,UAAU,CAEtB;CACF;AAED;;;;;;;GAOG;AACH,wBAAgB,eAAe,CAC7B,OAAO,EAAE,SAAS,eAAe,EAAE,GAClC,UAAU,CAIZ"}
|
|
@@ -0,0 +1,142 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Raw corpus counts, kept beside the quantised table rather than inside it.
|
|
3
|
+
*
|
|
4
|
+
* {@link import("./frequency-table.js").FrequencyTable} packs a count into four
|
|
5
|
+
* bits, because 16 buckets are all the decoder compares and they cost 9 KB
|
|
6
|
+
* where raw single-byte counts cost 23 KB. A caller ranking words against each
|
|
7
|
+
* other needs what that packing throws away. Rank the 120,858 CC-CEDICT
|
|
8
|
+
* headwords the full tier holds by `cost` and 5,934 of them land on the value
|
|
9
|
+
* at rank 10,000, so which words make a top-10,000 cut is decided by codepoint
|
|
10
|
+
* order. Rank the same words on the counts those buckets were quantised from
|
|
11
|
+
* and the tie at that rank is 16 words wide.
|
|
12
|
+
*
|
|
13
|
+
* So this is a second artifact, read by a caller that asks for it and by
|
|
14
|
+
* nothing on the decoding path. See `docs/dictionaries/` for the shape of it.
|
|
15
|
+
*/
|
|
16
|
+
import { claimKeys, orderedClaims } from "./artifact-claims.js";
|
|
17
|
+
/**
|
|
18
|
+
* The largest count that survives a round trip, which is 32 bits of varint.
|
|
19
|
+
*
|
|
20
|
+
* jieba's busiest word is attested 883,634 times, which leaves the ceiling
|
|
21
|
+
* three orders of magnitude clear of anything the corpus holds. It bounds the
|
|
22
|
+
* encoder rather than rejecting data.
|
|
23
|
+
*/
|
|
24
|
+
const MAX_COUNT = 0xff_ff_ff_ff;
|
|
25
|
+
const CONTINUATION = 0x80;
|
|
26
|
+
const PAYLOAD = 0x7f;
|
|
27
|
+
const SHIFT = 7;
|
|
28
|
+
/**
|
|
29
|
+
* Corpus counts for every key of a tier, one variable-length integer each.
|
|
30
|
+
*
|
|
31
|
+
* Counts are positional in the same sense the frequency table is: entry *n*
|
|
32
|
+
* describes the key at position *n* of the matching
|
|
33
|
+
* {@link import("./key-index.js").KeyIndex}, which is what
|
|
34
|
+
* {@link import("./dictionary.js").Dictionary.wordAt} reads.
|
|
35
|
+
*
|
|
36
|
+
* The encoding is LEB128, measured against the alternatives on the full tier's
|
|
37
|
+
* 723,147 keys. It takes 747 KB, and 243 KB brotli, where a fixed 32-bit array
|
|
38
|
+
* takes 2,825 KB and 299 KB and decimal text takes 1,586 KB and 281 KB. Two
|
|
39
|
+
* thirds of the keys are attested nowhere and most of the rest carry a small
|
|
40
|
+
* count, and a fixed width pays for the maximum on all of them.
|
|
41
|
+
*
|
|
42
|
+
* Unlike the rest of the artifact this *is* parsed on load, into one typed
|
|
43
|
+
* array. Nothing in a browser reads it, so the load cost buys a random access
|
|
44
|
+
* that a varint stream cannot otherwise give.
|
|
45
|
+
*/
|
|
46
|
+
export class WordCounts {
|
|
47
|
+
/**
|
|
48
|
+
* Encode raw counts, in key order.
|
|
49
|
+
*
|
|
50
|
+
* A count of zero means the word is attested nowhere in the corpus, which is
|
|
51
|
+
* a fact about the word and is stored as one. Fractions and negatives are
|
|
52
|
+
* floored into range rather than rejected, since a source that supplies one
|
|
53
|
+
* has already been through the merge.
|
|
54
|
+
*/
|
|
55
|
+
static build(counts) {
|
|
56
|
+
const values = Uint32Array.from(counts, (count) => Number.isFinite(count) ? Math.min(MAX_COUNT, Math.max(0, count)) : 0);
|
|
57
|
+
const packed = [];
|
|
58
|
+
for (const value of values) {
|
|
59
|
+
let remaining = value;
|
|
60
|
+
do {
|
|
61
|
+
const byte = remaining & PAYLOAD;
|
|
62
|
+
remaining >>>= SHIFT;
|
|
63
|
+
packed.push(remaining > 0 ? byte | CONTINUATION : byte);
|
|
64
|
+
} while (remaining > 0);
|
|
65
|
+
}
|
|
66
|
+
return new WordCounts(values, Uint8Array.from(packed));
|
|
67
|
+
}
|
|
68
|
+
/**
|
|
69
|
+
* Decode packed bytes read back from an artifact.
|
|
70
|
+
*
|
|
71
|
+
* How many counts there are is a property of the stream rather than an
|
|
72
|
+
* argument, since every count ends at the first byte without a continuation
|
|
73
|
+
* bit. A truncated stream ends mid-count, and the trailing bytes are dropped
|
|
74
|
+
* rather than reported as a value the file never held.
|
|
75
|
+
*/
|
|
76
|
+
static from(packed) {
|
|
77
|
+
const values = [];
|
|
78
|
+
let value = 0;
|
|
79
|
+
let shift = 0;
|
|
80
|
+
for (const byte of packed) {
|
|
81
|
+
// `+` rather than `|`, which is signed and would wrap the top bit of a
|
|
82
|
+
// 32-bit count into a negative number.
|
|
83
|
+
value += (byte & PAYLOAD) * 2 ** shift;
|
|
84
|
+
if ((byte & CONTINUATION) === 0) {
|
|
85
|
+
values.push(value);
|
|
86
|
+
value = 0;
|
|
87
|
+
shift = 0;
|
|
88
|
+
}
|
|
89
|
+
else {
|
|
90
|
+
shift += SHIFT;
|
|
91
|
+
}
|
|
92
|
+
}
|
|
93
|
+
return new WordCounts(Uint32Array.from(values), packed);
|
|
94
|
+
}
|
|
95
|
+
#counts;
|
|
96
|
+
#packed;
|
|
97
|
+
/**
|
|
98
|
+
* Hold decoded counts beside the bytes they came from. Use
|
|
99
|
+
* {@link WordCounts.build} or {@link WordCounts.from}.
|
|
100
|
+
*/
|
|
101
|
+
constructor(counts, packed) {
|
|
102
|
+
this.#counts = counts;
|
|
103
|
+
this.#packed = packed;
|
|
104
|
+
}
|
|
105
|
+
/**
|
|
106
|
+
* How many keys the counts describe.
|
|
107
|
+
*
|
|
108
|
+
* Worth checking against {@link import("./dictionary.js").Dictionary.size}
|
|
109
|
+
* before pairing the two by position. The counts are built for one tier, and
|
|
110
|
+
* against any other tier every position would name a different word.
|
|
111
|
+
*/
|
|
112
|
+
get size() {
|
|
113
|
+
return this.#counts.length;
|
|
114
|
+
}
|
|
115
|
+
/**
|
|
116
|
+
* Corpus occurrences of the key at a position.
|
|
117
|
+
*
|
|
118
|
+
* Positions outside the table report zero, which is what an unattested word
|
|
119
|
+
* reports, so a caller ranking by count sorts a missing key to the bottom.
|
|
120
|
+
*/
|
|
121
|
+
countOf(at) {
|
|
122
|
+
return this.#counts[at] ?? 0;
|
|
123
|
+
}
|
|
124
|
+
/**
|
|
125
|
+
* The packed bytes, ready to be written out.
|
|
126
|
+
*/
|
|
127
|
+
serialise() {
|
|
128
|
+
return this.#packed;
|
|
129
|
+
}
|
|
130
|
+
}
|
|
131
|
+
/**
|
|
132
|
+
* Compile the counts artifact from merged entries.
|
|
133
|
+
*
|
|
134
|
+
* The keys are claimed and ordered the way `artifact.ts` claims and orders
|
|
135
|
+
* them, so position *n* here is position *n* there. Only the `full` tier is
|
|
136
|
+
* written: a caller ranking words wants the whole vocabulary ordered, and a
|
|
137
|
+
* ranking over part of it answers a different question.
|
|
138
|
+
*/
|
|
139
|
+
export function buildWordCounts(entries) {
|
|
140
|
+
return WordCounts.build(orderedClaims(claimKeys(entries)).map(([, entry]) => entry.frequency));
|
|
141
|
+
}
|
|
142
|
+
//# sourceMappingURL=word-counts.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"word-counts.js","sourceRoot":"","sources":["../../src/dictionary/word-counts.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;GAcG;AACH,OAAO,EAAE,SAAS,EAAE,aAAa,EAAE,MAAM,sBAAsB,CAAC;AAGhE;;;;;;GAMG;AACH,MAAM,SAAS,GAAG,aAAa,CAAC;AAEhC,MAAM,YAAY,GAAG,IAAI,CAAC;AAC1B,MAAM,OAAO,GAAG,IAAI,CAAC;AACrB,MAAM,KAAK,GAAG,CAAC,CAAC;AAEhB;;;;;;;;;;;;;;;;;GAiBG;AACH,MAAM,OAAO,UAAU;IACrB;;;;;;;OAOG;IACH,MAAM,CAAC,KAAK,CAAC,MAAyB;QACpC,MAAM,MAAM,GAAG,WAAW,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC,KAAK,EAAE,EAAE,CAChD,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,SAAS,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CACrE,CAAC;QAEF,MAAM,MAAM,GAAa,EAAE,CAAC;QAC5B,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;YAC3B,IAAI,SAAS,GAAG,KAAK,CAAC;YACtB,GAAG,CAAC;gBACF,MAAM,IAAI,GAAG,SAAS,GAAG,OAAO,CAAC;gBACjC,SAAS,MAAM,KAAK,CAAC;gBACrB,MAAM,CAAC,IAAI,CAAC,SAAS,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,GAAG,YAAY,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;YAC1D,CAAC,QAAQ,SAAS,GAAG,CAAC,EAAE;QAC1B,CAAC;QAED,OAAO,IAAI,UAAU,CAAC,MAAM,EAAE,UAAU,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC;IACzD,CAAC;IAED;;;;;;;OAOG;IACH,MAAM,CAAC,IAAI,CAAC,MAAkB;QAC5B,MAAM,MAAM,GAAa,EAAE,CAAC;QAC5B,IAAI,KAAK,GAAG,CAAC,CAAC;QACd,IAAI,KAAK,GAAG,CAAC,CAAC;QACd,KAAK,MAAM,IAAI,IAAI,MAAM,EAAE,CAAC;YAC1B,uEAAuE;YACvE,uCAAuC;YACvC,KAAK,IAAI,CAAC,IAAI,GAAG,OAAO,CAAC,GAAG,CAAC,IAAI,KAAK,CAAC;YACvC,IAAI,CAAC,IAAI,GAAG,YAAY,CAAC,KAAK,CAAC,EAAE,CAAC;gBAChC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;gBACnB,KAAK,GAAG,CAAC,CAAC;gBACV,KAAK,GAAG,CAAC,CAAC;YACZ,CAAC;iBAAM,CAAC;gBACN,KAAK,IAAI,KAAK,CAAC;YACjB,CAAC;QACH,CAAC;QAED,OAAO,IAAI,UAAU,CAAC,WAAW,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,MAAM,CAAC,CAAC;IAC1D,CAAC;IAEQ,OAAO,CAAc;IACrB,OAAO,CAAa;IAE7B;;;OAGG;IACH,YAAoB,MAAmB,EAAE,MAAkB;QACzD,IAAI,CAAC,OAAO,GAAG,MAAM,CAAC;QACtB,IAAI,CAAC,OAAO,GAAG,MAAM,CAAC;IACxB,CAAC;IAED;;;;;;OAMG;IACH,IAAI,IAAI;QACN,OAAO,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC;IAC7B,CAAC;IAED;;;;;OAKG;IACH,OAAO,CAAC,EAAU;QAChB,OAAO,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC,IAAI,CAAC,CAAC;IAC/B,CAAC;IAED;;OAEG;IACH,SAAS;QACP,OAAO,IAAI,CAAC,OAAO,CAAC;IACtB,CAAC;CACF;AAED;;;;;;;GAOG;AACH,MAAM,UAAU,eAAe,CAC7B,OAAmC;IAEnC,OAAO,UAAU,CAAC,KAAK,CACrB,aAAa,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,KAAK,CAAC,EAAE,EAAE,CAAC,KAAK,CAAC,SAAS,CAAC,CACtE,CAAC;AACJ,CAAC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"annotate.d.ts","sourceRoot":"","sources":["../../src/format/annotate.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AACH,OAAO,EAAU,KAAK,WAAW,
|
|
1
|
+
{"version":3,"file":"annotate.d.ts","sourceRoot":"","sources":["../../src/format/annotate.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AACH,OAAO,EAAU,KAAK,WAAW,EAAuB,MAAM,WAAW,CAAC;AAC1E,OAAO,EAAE,KAAK,cAAc,EAAiB,MAAM,sBAAsB,CAAC;AAC1E,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,6BAA6B,CAAC;AAE9D;;;;;;;;;;;;;;;GAeG;AACH,eAAO,MAAM,QAAQ,OAAO,CAAC;AAE7B;;GAEG;AACH,MAAM,WAAW,UAAU;IACzB,mEAAmE;IACnE,QAAQ,CAAC,MAAM,EAAE,MAAM,GAAG,SAAS,CAAC;IACpC,QAAQ,CAAC,MAAM,EAAE,SAAS,cAAc,EAAE,CAAC;CAC5C;AAED;;;;;;;;GAQG;AACH,wBAAgB,aAAa,CAAC,MAAM,EAAE,SAAS,cAAc,EAAE,GAAG,UAAU,EAAE,CAc7E;AA2ED;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AACH,wBAAgB,eAAe,CAC7B,MAAM,EAAE,SAAS,cAAc,EAAE,EACjC,OAAO,GAAE,WAAgB,GACxB,MAAM,CAIR;AAED;;;;;GAKG;AACH,wBAAgB,sBAAsB,CACpC,UAAU,EAAE,UAAU,EACtB,IAAI,EAAE,MAAM,EACZ,OAAO,GAAE,WAAgB,GACxB,MAAM,CAER;AAED;;;;;;;;;;;GAWG;AACH,wBAAgB,aAAa,CAC3B,UAAU,EAAE,UAAU,EACtB,IAAI,EAAE,MAAM,EACZ,OAAO,GAAE,WAAgB,GACxB,MAAM,CAER"}
|
package/dist/format/annotate.js
CHANGED
|
@@ -4,7 +4,7 @@
|
|
|
4
4
|
* The pinyin sits above the characters rather than beside them, so the pieces
|
|
5
5
|
* are regrouped against the text they annotate.
|
|
6
6
|
*/
|
|
7
|
-
import { escape, toHtml,
|
|
7
|
+
import { escape, toHtml, writePieces } from "./html.js";
|
|
8
8
|
import { convertPieces } from "../decode/convert.js";
|
|
9
9
|
/**
|
|
10
10
|
* What the characters being annotated declare themselves to be.
|
|
@@ -101,9 +101,9 @@ function writeAnnotation(annotation, options) {
|
|
|
101
101
|
// characters — but {@link toAnnotatedHtml} takes pieces from the caller,
|
|
102
102
|
// and there is no base to hang an annotation on, so it is written as the
|
|
103
103
|
// plain markup it would have got from {@link toHtml}.
|
|
104
|
-
return pieces
|
|
104
|
+
return writePieces(pieces, options);
|
|
105
105
|
}
|
|
106
|
-
const reading = pieces
|
|
106
|
+
const reading = writePieces(pieces, options);
|
|
107
107
|
// The tag goes on the <ruby> rather than around the base, and the syllables
|
|
108
108
|
// inside the <rt> override it with their own. `<rb>` would be the obvious
|
|
109
109
|
// place for it and is not an element any more: the WHATWG parser drops it,
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"annotate.js","sourceRoot":"","sources":["../../src/format/annotate.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AACH,OAAO,EAAE,MAAM,EAAoB,MAAM,EAAE,
|
|
1
|
+
{"version":3,"file":"annotate.js","sourceRoot":"","sources":["../../src/format/annotate.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AACH,OAAO,EAAE,MAAM,EAAoB,MAAM,EAAE,WAAW,EAAE,MAAM,WAAW,CAAC;AAC1E,OAAO,EAAuB,aAAa,EAAE,MAAM,sBAAsB,CAAC;AAG1E;;;;;;;;;;;;;;;GAeG;AACH,MAAM,CAAC,MAAM,QAAQ,GAAG,IAAI,CAAC;AAW7B;;;;;;;;GAQG;AACH,MAAM,UAAU,aAAa,CAAC,MAAiC;IAC7D,MAAM,MAAM,GAAiB,EAAE,CAAC;IAChC,KAAK,MAAM,CAAC,EAAE,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,EAAE,EAAE,CAAC;QAC3C,MAAM,IAAI,GAAG,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;QAC3B,IAAI,IAAI,EAAE,MAAM,KAAK,SAAS,IAAI,SAAS,CAAC,KAAK,EAAE,MAAM,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC;YACnE,MAAM,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,GAAG;gBAC1B,MAAM,EAAE,IAAI,CAAC,MAAM;gBACnB,MAAM,EAAE,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,KAAK,CAAC;aAChC,CAAC;YACF,SAAS;QACX,CAAC;QACD,MAAM,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,KAAK,CAAC,MAAM,EAAE,MAAM,EAAE,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;IACzD,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC;AAED;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AACH,SAAS,SAAS,CAChB,KAAqB,EACrB,IAAgC;IAEhC,IAAI,KAAK,CAAC,QAAQ,KAAK,SAAS,EAAE,CAAC;QACjC,OAAO,KAAK,CAAC,MAAM,KAAK,SAAS,CAAC;IACpC,CAAC;IACD,IAAI,KAAK,CAAC,MAAM,KAAK,SAAS,EAAE,CAAC;QAC/B,OAAO,KAAK,CAAC;IACf,CAAC;IACD,OAAO,CACL,KAAK,CAAC,IAAI,CAAC,IAAI,EAAE,KAAK,EAAE;QACxB,CAAC,IAAI,EAAE,QAAQ,KAAK,SAAS,IAAI,IAAI,CAAC,MAAM,KAAK,SAAS,CAAC,CAC5D,CAAC;AACJ,CAAC;AAED;;GAEG;AACH,SAAS,eAAe,CAAC,UAAsB,EAAE,OAAoB;IACnE,MAAM,EAAE,IAAI,GAAG,IAAI,EAAE,GAAG,OAAO,CAAC;IAChC,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,GAAG,UAAU,CAAC;IACtC,IAAI,MAAM,CAAC,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,QAAQ,KAAK,SAAS,CAAC,EAAE,CAAC;QAC1D,oEAAoE;QACpE,2EAA2E;QAC3E,uEAAuE;QACvE,wEAAwE;QACxE,qDAAqD;QACrD,OAAO,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;IACpD,CAAC;IACD,IAAI,MAAM,KAAK,SAAS,EAAE,CAAC;QACzB,wEAAwE;QACxE,yEAAyE;QACzE,yEAAyE;QACzE,yEAAyE;QACzE,sDAAsD;QACtD,OAAO,WAAW,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IACtC,CAAC;IAED,MAAM,OAAO,GAAG,WAAW,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAC7C,4EAA4E;IAC5E,0EAA0E;IAC1E,2EAA2E;IAC3E,2EAA2E;IAC3E,yBAAyB;IACzB,MAAM,OAAO,GAAG,IAAI,CAAC,CAAC,CAAC,eAAe,QAAQ,IAAI,CAAC,CAAC,CAAC,QAAQ,CAAC;IAC9D,OAAO,GAAG,OAAO,GAAG,MAAM,CAAC,MAAM,CAAC,iBAAiB,OAAO,wBAAwB,CAAC;AACrF,CAAC;AAED;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AACH,MAAM,UAAU,eAAe,CAC7B,MAAiC,EACjC,OAAO,GAAgB,EAAE;IAEzB,OAAO,aAAa,CAAC,MAAM,CAAC;SACzB,GAAG,CAAC,CAAC,UAAU,EAAE,EAAE,CAAC,eAAe,CAAC,UAAU,EAAE,OAAO,CAAC,CAAC;SACzD,IAAI,CAAC,EAAE,CAAC,CAAC;AACd,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,sBAAsB,CACpC,UAAsB,EACtB,IAAY,EACZ,OAAO,GAAgB,EAAE;IAEzB,OAAO,eAAe,CAAC,aAAa,CAAC,UAAU,EAAE,IAAI,EAAE,OAAO,CAAC,EAAE,OAAO,CAAC,CAAC;AAC5E,CAAC;AAED;;;;;;;;;;;GAWG;AACH,MAAM,UAAU,aAAa,CAC3B,UAAsB,EACtB,IAAY,EACZ,OAAO,GAAgB,EAAE;IAEzB,OAAO,MAAM,CAAC,aAAa,CAAC,UAAU,EAAE,IAAI,EAAE,OAAO,CAAC,EAAE,OAAO,CAAC,CAAC;AACnE,CAAC"}
|