@kensio/pinyinjs 1.7.9 → 1.7.10

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/data/core.freq CHANGED
Binary file
package/data/full.freq CHANGED
Binary file
@@ -22,7 +22,7 @@
22
22
  "readings": "full.entries",
23
23
  "frequencies": "full.freq",
24
24
  "bytes": 9143780,
25
- "compressedBytes": 2400058
25
+ "compressedBytes": 2402315
26
26
  }
27
27
  },
28
28
  "counts": {
Binary file
@@ -8,7 +8,7 @@
8
8
  import { toCharacters } from "../script/characters.js";
9
9
  export { claimKeys } from "./artifact-claims.js";
10
10
  export { findRoundTripFailure, readArtifact, readingsByKey, } from "./artifact-read.js";
11
- import { FrequencyTable } from "./frequency-table.js";
11
+ import { countForQuantising, FrequencyTable } from "./frequency-table.js";
12
12
  import { KeyIndex } from "./key-index.js";
13
13
  import { claimKeys, orderedClaims } from "./artifact-claims.js";
14
14
  import { ALTERNATE, BOUNDARY_SEPARATOR, COLUMN, encodeReading, LINE, PROPER_NOUN_FLAG, } from "./artifact-format.js";
@@ -92,7 +92,7 @@ export function buildArtifact(entries) {
92
92
  .map((alternate) => encodeReading(alternate))
93
93
  .join(ALTERNATE),
94
94
  ]));
95
- frequencies.push(entry.frequency);
95
+ frequencies.push(countForQuantising(entry.frequency, entry.isProperNoun));
96
96
  }
97
97
  return {
98
98
  keys: index.serialise(),
@@ -1 +1 @@
1
- {"version":3,"file":"artifact.js","sourceRoot":"","sources":["../../src/dictionary/artifact.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AACH,OAAO,EAAE,YAAY,EAAE,MAAM,yBAAyB,CAAC;AACvD,OAAO,EAAE,SAAS,EAAE,MAAM,sBAAsB,CAAC;AACjD,OAAO,EACL,oBAAoB,EACpB,YAAY,EACZ,aAAa,GACd,MAAM,oBAAoB,CAAC;AAE5B,OAAO,EAAE,cAAc,EAAE,MAAM,sBAAsB,CAAC;AACtD,OAAO,EAAE,QAAQ,EAAE,MAAM,gBAAgB,CAAC;AAC1C,OAAO,EAAE,SAAS,EAAE,aAAa,EAAE,MAAM,sBAAsB,CAAC;AAChE,OAAO,EACL,SAAS,EACT,kBAAkB,EAClB,MAAM,EAEN,aAAa,EACb,IAAI,EACJ,gBAAgB,GACjB,MAAM,sBAAsB,CAAC;AAE9B,OAAO,EACL,aAAa,EAEb,aAAa,EACb,gBAAgB,GACjB,MAAM,sBAAsB,CAAC;AAE9B;;GAEG;AACH,SAAS,iBAAiB,CACxB,OAA6C;IAE7C,MAAM,QAAQ,GAAG,IAAI,GAAG,EAAkB,CAAC;IAC3C,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,OAAO,EAAE,CAAC;QACnC,IAAI,YAAY,CAAC,GAAG,CAAC,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACnC,QAAQ,CAAC,GAAG,CAAC,GAAG,EAAE,aAAa,CAAC,KAAK,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,CAAC;QACtD,CAAC;IACH,CAAC;IACD,OAAO,QAAQ,CAAC;AAClB,CAAC;AAED;;;;;;;GAOG;AACH,SAAS,WAAW,CAClB,GAAW,EACX,OAAe,EACf,QAAqC;IAErC,MAAM,KAAK,GAAG,YAAY,CAAC,GAAG,CAAC,CAAC;IAChC,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACrB,OAAO,KAAK,CAAC;IACf,CAAC;IACD,MAAM,OAAO,GAAa,EAAE,CAAC;IAC7B,KAAK,MAAM,SAAS,IAAI,KAAK,EAAE,CAAC;QAC9B,MAAM,KAAK,GAAG,QAAQ,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;QACtC,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;YACxB,OAAO,KAAK,CAAC;QACf,CAAC;QACD,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IACtB,CAAC;IACD,OAAO,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,OAAO,CAAC;AACvC,CAAC;AAED;;GAEG;AACH,SAAS,WAAW,CAAC,OAA0B;IAC7C,IAAI,GAAG,GAAG,OAAO,CAAC,MAAM,CAAC;IACzB,OAAO,GAAG,GAAG,CAAC,IAAI,OAAO,CAAC,GAAG,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC;QAC1C,GAAG,EAAE,CAAC;IACR,CAAC;IACD,OAAO,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;AAC5C,CAAC;AAED;;;;;;;;;;;;GAYG;AACH,MAAM,UAAU,aAAa,CAC3B,OAAmC;IAEnC,MAAM,OAAO,GAAG,SAAS,CAAC,OAAO,CAAC,CAAC;IACnC,MAAM,QAAQ,GAAG,iBAAiB,CAAC,OAAO,CAAC,CAAC;IAC5C,MAAM,OAAO,GAAG,aAAa,CAAC,OAAO,CAAC,CAAC;IACvC,MAAM,KAAK,GAAG,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC;IAEpE,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,MAAM,WAAW,GAAa,EAAE,CAAC;IACjC,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,OAAO,EAAE,CAAC;QACnC,MAAM,OAAO,GAAG,aAAa,CAAC,KAAK,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC;QAEjD,KAAK,CAAC,IAAI,CACR,WAAW,CAAC;YACV,WAAW,CAAC,GAAG,EAAE,OAAO,EAAE,QAAQ,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,OAAO;YAClD,KAAK,CAAC,QAAQ,CAAC,EAAE,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,aAAa,CAAC,KAAK,CAAC,QAAQ,CAAC,EAAE,CAAC;YACvE,KAAK,CAAC,YAAY;YAClB,KAAK,CAAC,YAAY;gBAChB,CAAC,CAAC,gBAAgB;oBAChB,CAAC,KAAK,CAAC,cAAc,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,kBAAkB,CAAC;gBACvD,CAAC,CAAC,EAAE;YACN,CAAC,KAAK,CAAC,UAAU,IAAI,EAAE,CAAC;iBACrB,GAAG,CAAC,CAAC,SAAS,EAAE,EAAE,CAAC,aAAa,CAAC,SAAS,CAAC,CAAC;iBAC5C,IAAI,CAAC,SAAS,CAAC;SACnB,CAAC,CACH,CAAC;QACF,WAAW,CAAC,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC;IACpC,CAAC;IAED,OAAO;QACL,IAAI,EAAE,KAAK,CAAC,SAAS,EAAE;QACvB,OAAO,EAAE,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC;QACzB,WAAW,EAAE,cAAc,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC,SAAS,EAAE;KAC3D,CAAC;AACJ,CAAC"}
1
+ {"version":3,"file":"artifact.js","sourceRoot":"","sources":["../../src/dictionary/artifact.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AACH,OAAO,EAAE,YAAY,EAAE,MAAM,yBAAyB,CAAC;AACvD,OAAO,EAAE,SAAS,EAAE,MAAM,sBAAsB,CAAC;AACjD,OAAO,EACL,oBAAoB,EACpB,YAAY,EACZ,aAAa,GACd,MAAM,oBAAoB,CAAC;AAE5B,OAAO,EAAE,kBAAkB,EAAE,cAAc,EAAE,MAAM,sBAAsB,CAAC;AAC1E,OAAO,EAAE,QAAQ,EAAE,MAAM,gBAAgB,CAAC;AAC1C,OAAO,EAAE,SAAS,EAAE,aAAa,EAAE,MAAM,sBAAsB,CAAC;AAChE,OAAO,EACL,SAAS,EACT,kBAAkB,EAClB,MAAM,EAEN,aAAa,EACb,IAAI,EACJ,gBAAgB,GACjB,MAAM,sBAAsB,CAAC;AAE9B,OAAO,EACL,aAAa,EAEb,aAAa,EACb,gBAAgB,GACjB,MAAM,sBAAsB,CAAC;AAE9B;;GAEG;AACH,SAAS,iBAAiB,CACxB,OAA6C;IAE7C,MAAM,QAAQ,GAAG,IAAI,GAAG,EAAkB,CAAC;IAC3C,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,OAAO,EAAE,CAAC;QACnC,IAAI,YAAY,CAAC,GAAG,CAAC,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACnC,QAAQ,CAAC,GAAG,CAAC,GAAG,EAAE,aAAa,CAAC,KAAK,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,CAAC;QACtD,CAAC;IACH,CAAC;IACD,OAAO,QAAQ,CAAC;AAClB,CAAC;AAED;;;;;;;GAOG;AACH,SAAS,WAAW,CAClB,GAAW,EACX,OAAe,EACf,QAAqC;IAErC,MAAM,KAAK,GAAG,YAAY,CAAC,GAAG,CAAC,CAAC;IAChC,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACrB,OAAO,KAAK,CAAC;IACf,CAAC;IACD,MAAM,OAAO,GAAa,EAAE,CAAC;IAC7B,KAAK,MAAM,SAAS,IAAI,KAAK,EAAE,CAAC;QAC9B,MAAM,KAAK,GAAG,QAAQ,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;QACtC,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;YACxB,OAAO,KAAK,CAAC;QACf,CAAC;QACD,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IACtB,CAAC;IACD,OAAO,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,OAAO,CAAC;AACvC,CAAC;AAED;;GAEG;AACH,SAAS,WAAW,CAAC,OAA0B;IAC7C,IAAI,GAAG,GAAG,OAAO,CAAC,MAAM,CAAC;IACzB,OAAO,GAAG,GAAG,CAAC,IAAI,OAAO,CAAC,GAAG,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC;QAC1C,GAAG,EAAE,CAAC;IACR,CAAC;IACD,OAAO,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;AAC5C,CAAC;AAED;;;;;;;;;;;;GAYG;AACH,MAAM,UAAU,aAAa,CAC3B,OAAmC;IAEnC,MAAM,OAAO,GAAG,SAAS,CAAC,OAAO,CAAC,CAAC;IACnC,MAAM,QAAQ,GAAG,iBAAiB,CAAC,OAAO,CAAC,CAAC;IAC5C,MAAM,OAAO,GAAG,aAAa,CAAC,OAAO,CAAC,CAAC;IACvC,MAAM,KAAK,GAAG,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC;IAEpE,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,MAAM,WAAW,GAAa,EAAE,CAAC;IACjC,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,OAAO,EAAE,CAAC;QACnC,MAAM,OAAO,GAAG,aAAa,CAAC,KAAK,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC;QAEjD,KAAK,CAAC,IAAI,CACR,WAAW,CAAC;YACV,WAAW,CAAC,GAAG,EAAE,OAAO,EAAE,QAAQ,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,OAAO;YAClD,KAAK,CAAC,QAAQ,CAAC,EAAE,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,aAAa,CAAC,KAAK,CAAC,QAAQ,CAAC,EAAE,CAAC;YACvE,KAAK,CAAC,YAAY;YAClB,KAAK,CAAC,YAAY;gBAChB,CAAC,CAAC,gBAAgB;oBAChB,CAAC,KAAK,CAAC,cAAc,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,kBAAkB,CAAC;gBACvD,CAAC,CAAC,EAAE;YACN,CAAC,KAAK,CAAC,UAAU,IAAI,EAAE,CAAC;iBACrB,GAAG,CAAC,CAAC,SAAS,EAAE,EAAE,CAAC,aAAa,CAAC,SAAS,CAAC,CAAC;iBAC5C,IAAI,CAAC,SAAS,CAAC;SACnB,CAAC,CACH,CAAC;QACF,WAAW,CAAC,IAAI,CAAC,kBAAkB,CAAC,KAAK,CAAC,SAAS,EAAE,KAAK,CAAC,YAAY,CAAC,CAAC,CAAC;IAC5E,CAAC;IAED,OAAO;QACL,IAAI,EAAE,KAAK,CAAC,SAAS,EAAE;QACvB,OAAO,EAAE,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC;QACzB,WAAW,EAAE,cAAc,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC,SAAS,EAAE;KAC3D,CAAC;AACJ,CAAC"}
@@ -46,6 +46,60 @@ export declare const FREQUENCY_BUCKETS = 16;
46
46
  * bucket sum, re-broken by the order the floats happened to be added in.
47
47
  */
48
48
  export declare const WORD_CHARGE = 4.62;
49
+ /**
50
+ * The count a name the corpus never counted is quantised at.
51
+ *
52
+ * A count of zero earns bucket 0, and {@link FrequencyTable.costOf} reads that
53
+ * as `−log P` with `P` of `1/N`. On jieba's corpus the flat cost it produces is
54
+ * 19.62, and two mid-frequency characters come to less. 脸书 cost 19.62 where
55
+ * 脸 and 书 together cost 18.24, and `decodeRun` read Facebook as a face and a
56
+ * book. 推特, 高雄, 乐高 and 网飞 all went the same way, and 671 of the 5,039
57
+ * uncounted names of two or three characters came apart when read on their
58
+ * own. 谷歌 survived on an exact tie.
59
+ *
60
+ * **Zero is the wrong count for a name jieba's list never held.** The list has
61
+ * no frequency floor to be under. It carries 一万一千五百二十颗 on two
62
+ * occurrences. A word missing from it was left out of the lexicon, and the
63
+ * corpus was never asked about it. Names are where that bites, since a name
64
+ * enters the language whenever something is given one. 微软 was counted 1757
65
+ * times and 脸书 not at all, and only one of the two existed when the list was
66
+ * drawn. 10,676 of the dictionary's 42,861 names are counted nowhere.
67
+ *
68
+ * **3 is jieba's own answer for a word it lists and cannot count.** Its counts
69
+ * bottom out at 2, and 3 carries 159,318 of 349,045 entries against 40,502 at 2
70
+ * and 12,679 at 4. Interpolating between the neighbours accounts for about
71
+ * 22,000 of those, and the other 137,000 carry a default. It is the default a
72
+ * hand-added entry gets (AT&T, B超 and C++ are all counted 3).
73
+ *
74
+ * The value is loose. Bucket 2 runs from 3 to 8, so any count jieba might have
75
+ * settled on in that range quantises the same way and lands the cost at 17.62.
76
+ * What the floor buys a name is two buckets against a split.
77
+ *
78
+ * **Names only.** The same floor over every uncounted key lifts two thirds of
79
+ * the dictionary by two buckets and re-tips
80
+ * {@link import("../decode/lattice-types.js").READING_CHARGE}, which was sized
81
+ * against bucket 0 at 19.62. Measured that way 從容地 joins, 都會 reads `dūhuì`
82
+ * and 過得 reads `guò dé`, and the gold harness goes from 3 misses to 6. The
83
+ * dictionary's tail is where 费耗, 块肉 and 拍张 live, and bucket 0 is what
84
+ * holds them off.
85
+ *
86
+ * 428 of the 671 names now stay whole. The ones left are compounds a split
87
+ * suits, 中国菜, 日本人, 上海话 and 美国人 among them, and each beats its own
88
+ * entry by more than two buckets. Over the 88,866 lines of Tatoeba and
89
+ * zh.wikipedia 86 of 139,682 Han runs move, the gold harness holds at 3 misses
90
+ * and CPP holds at 91.49%.
91
+ */
92
+ export declare const UNCOUNTED_NAME = 3;
93
+ /**
94
+ * The count an entry is quantised at. Its own, unless it is a name the corpus
95
+ * never counted. See {@link UNCOUNTED_NAME}.
96
+ *
97
+ * Applied where the artifact is written and nowhere else, so `full.counts`
98
+ * keeps the corpus as it was met. The two files disagree by design over these
99
+ * 10,676 keys, and `data.test.ts` checks their alignment through this
100
+ * function.
101
+ */
102
+ export declare function countForQuantising(frequency: number, isProperNoun: boolean): number;
49
103
  /**
50
104
  * The charge a corpus derives, for the build to hold {@link WORD_CHARGE} to.
51
105
  *
@@ -71,6 +125,9 @@ export declare class FrequencyTable {
71
125
  * the word was not attested, which still earns the lowest bucket rather than
72
126
  * being excluded — an unattested word in the dictionary is rare, not
73
127
  * impossible.
128
+ *
129
+ * Counts arrive through {@link countForQuantising}, which is where a name the
130
+ * corpus never counted is given jieba's own default for one.
74
131
  */
75
132
  static build(frequencies: readonly number[]): FrequencyTable;
76
133
  /**
@@ -1 +1 @@
1
- {"version":3,"file":"frequency-table.d.ts","sourceRoot":"","sources":["../../src/dictionary/frequency-table.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AACH,eAAO,MAAM,iBAAiB,KAAK,CAAC;AAIpC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAoCG;AACH,eAAO,MAAM,WAAW,OAAO,CAAC;AAEhC;;;;;;GAMG;AACH,wBAAgB,iBAAiB,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,GAAG,MAAM,CAExE;AAED;;;;;;GAMG;AACH,qBAAa,cAAc;;IACzB;;;;;;;;OAQG;IACH,MAAM,CAAC,KAAK,CAAC,WAAW,EAAE,SAAS,MAAM,EAAE,GAAG,cAAc,CA4B3D;IAED;;OAEG;IACH,MAAM,CAAC,IAAI,CAAC,MAAM,EAAE,UAAU,EAAE,IAAI,EAAE,MAAM,GAAG,cAAc,CAE5D;IAKD;;;OAGG;IACH,OAAO,eAGN;IAED;;OAEG;IACH,IAAI,IAAI,IAAI,MAAM,CAEjB;IAED;;;;;OAKG;IACH,QAAQ,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAM9B;IAED;;;;;;;;OAQG;IACH,MAAM,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAE5B;IAED;;OAEG;IACH,SAAS,IAAI,UAAU,CAEtB;CACF"}
1
+ {"version":3,"file":"frequency-table.d.ts","sourceRoot":"","sources":["../../src/dictionary/frequency-table.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AACH,eAAO,MAAM,iBAAiB,KAAK,CAAC;AAIpC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAoCG;AACH,eAAO,MAAM,WAAW,OAAO,CAAC;AAEhC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA0CG;AACH,eAAO,MAAM,cAAc,IAAI,CAAC;AAEhC;;;;;;;;GAQG;AACH,wBAAgB,kBAAkB,CAChC,SAAS,EAAE,MAAM,EACjB,YAAY,EAAE,OAAO,GACpB,MAAM,CAER;AAED;;;;;;GAMG;AACH,wBAAgB,iBAAiB,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,GAAG,MAAM,CAExE;AAED;;;;;;GAMG;AACH,qBAAa,cAAc;;IACzB;;;;;;;;;;;OAWG;IACH,MAAM,CAAC,KAAK,CAAC,WAAW,EAAE,SAAS,MAAM,EAAE,GAAG,cAAc,CA4B3D;IAED;;OAEG;IACH,MAAM,CAAC,IAAI,CAAC,MAAM,EAAE,UAAU,EAAE,IAAI,EAAE,MAAM,GAAG,cAAc,CAE5D;IAKD;;;OAGG;IACH,OAAO,eAGN;IAED;;OAEG;IACH,IAAI,IAAI,IAAI,MAAM,CAEjB;IAED;;;;;OAKG;IACH,QAAQ,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAM9B;IAED;;;;;;;;OAQG;IACH,MAAM,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAE5B;IAED;;OAEG;IACH,SAAS,IAAI,UAAU,CAEtB;CACF"}
@@ -47,6 +47,62 @@ const MAX_BUCKET = FREQUENCY_BUCKETS - 1;
47
47
  * bucket sum, re-broken by the order the floats happened to be added in.
48
48
  */
49
49
  export const WORD_CHARGE = 4.62;
50
+ /**
51
+ * The count a name the corpus never counted is quantised at.
52
+ *
53
+ * A count of zero earns bucket 0, and {@link FrequencyTable.costOf} reads that
54
+ * as `−log P` with `P` of `1/N`. On jieba's corpus the flat cost it produces is
55
+ * 19.62, and two mid-frequency characters come to less. 脸书 cost 19.62 where
56
+ * 脸 and 书 together cost 18.24, and `decodeRun` read Facebook as a face and a
57
+ * book. 推特, 高雄, 乐高 and 网飞 all went the same way, and 671 of the 5,039
58
+ * uncounted names of two or three characters came apart when read on their
59
+ * own. 谷歌 survived on an exact tie.
60
+ *
61
+ * **Zero is the wrong count for a name jieba's list never held.** The list has
62
+ * no frequency floor to be under. It carries 一万一千五百二十颗 on two
63
+ * occurrences. A word missing from it was left out of the lexicon, and the
64
+ * corpus was never asked about it. Names are where that bites, since a name
65
+ * enters the language whenever something is given one. 微软 was counted 1757
66
+ * times and 脸书 not at all, and only one of the two existed when the list was
67
+ * drawn. 10,676 of the dictionary's 42,861 names are counted nowhere.
68
+ *
69
+ * **3 is jieba's own answer for a word it lists and cannot count.** Its counts
70
+ * bottom out at 2, and 3 carries 159,318 of 349,045 entries against 40,502 at 2
71
+ * and 12,679 at 4. Interpolating between the neighbours accounts for about
72
+ * 22,000 of those, and the other 137,000 carry a default. It is the default a
73
+ * hand-added entry gets (AT&T, B超 and C++ are all counted 3).
74
+ *
75
+ * The value is loose. Bucket 2 runs from 3 to 8, so any count jieba might have
76
+ * settled on in that range quantises the same way and lands the cost at 17.62.
77
+ * What the floor buys a name is two buckets against a split.
78
+ *
79
+ * **Names only.** The same floor over every uncounted key lifts two thirds of
80
+ * the dictionary by two buckets and re-tips
81
+ * {@link import("../decode/lattice-types.js").READING_CHARGE}, which was sized
82
+ * against bucket 0 at 19.62. Measured that way 從容地 joins, 都會 reads `dūhuì`
83
+ * and 過得 reads `guò dé`, and the gold harness goes from 3 misses to 6. The
84
+ * dictionary's tail is where 费耗, 块肉 and 拍张 live, and bucket 0 is what
85
+ * holds them off.
86
+ *
87
+ * 428 of the 671 names now stay whole. The ones left are compounds a split
88
+ * suits, 中国菜, 日本人, 上海话 and 美国人 among them, and each beats its own
89
+ * entry by more than two buckets. Over the 88,866 lines of Tatoeba and
90
+ * zh.wikipedia 86 of 139,682 Han runs move, the gold harness holds at 3 misses
91
+ * and CPP holds at 91.49%.
92
+ */
93
+ export const UNCOUNTED_NAME = 3;
94
+ /**
95
+ * The count an entry is quantised at. Its own, unless it is a name the corpus
96
+ * never counted. See {@link UNCOUNTED_NAME}.
97
+ *
98
+ * Applied where the artifact is written and nowhere else, so `full.counts`
99
+ * keeps the corpus as it was met. The two files disagree by design over these
100
+ * 10,676 keys, and `data.test.ts` checks their alignment through this
101
+ * function.
102
+ */
103
+ export function countForQuantising(frequency, isProperNoun) {
104
+ return isProperNoun && frequency <= 0 ? UNCOUNTED_NAME : frequency;
105
+ }
50
106
  /**
51
107
  * The charge a corpus derives, for the build to hold {@link WORD_CHARGE} to.
52
108
  *
@@ -73,6 +129,9 @@ export class FrequencyTable {
73
129
  * the word was not attested, which still earns the lowest bucket rather than
74
130
  * being excluded — an unattested word in the dictionary is rare, not
75
131
  * impossible.
132
+ *
133
+ * Counts arrive through {@link countForQuantising}, which is where a name the
134
+ * corpus never counted is given jieba's own default for one.
76
135
  */
77
136
  static build(frequencies) {
78
137
  // Looped rather than `Math.max(0, ...frequencies)`, which passes one
@@ -1 +1 @@
1
- {"version":3,"file":"frequency-table.js","sourceRoot":"","sources":["../../src/dictionary/frequency-table.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AACH,MAAM,CAAC,MAAM,iBAAiB,GAAG,EAAE,CAAC;AAEpC,MAAM,UAAU,GAAG,iBAAiB,GAAG,CAAC,CAAC;AAEzC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAoCG;AACH,MAAM,CAAC,MAAM,WAAW,GAAG,IAAI,CAAC;AAEhC;;;;;;GAMG;AACH,MAAM,UAAU,iBAAiB,CAAC,KAAa,EAAE,OAAe;IAC9D,OAAO,UAAU,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC;AAClE,CAAC;AAED;;;;;;GAMG;AACH,MAAM,OAAO,cAAc;IACzB;;;;;;;;OAQG;IACH,MAAM,CAAC,KAAK,CAAC,WAA8B;QACzC,qEAAqE;QACrE,4EAA4E;QAC5E,0EAA0E;QAC1E,IAAI,OAAO,GAAG,CAAC,CAAC;QAChB,KAAK,MAAM,SAAS,IAAI,WAAW,EAAE,CAAC;YACpC,IAAI,SAAS,GAAG,OAAO,EAAE,CAAC;gBACxB,OAAO,GAAG,SAAS,CAAC;YACtB,CAAC;QACH,CAAC;QACD,MAAM,KAAK,GAAG,OAAO,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,UAAU,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;QAEnE,MAAM,MAAM,GAAG,IAAI,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,WAAW,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC;QACjE,KAAK,MAAM,CAAC,KAAK,EAAE,SAAS,CAAC,IAAI,WAAW,CAAC,OAAO,EAAE,EAAE,CAAC;YACvD,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,SAAS,CAAC,CAAC;YACxC,MAAM,MAAM,GAAG,IAAI,CAAC,GAAG,CACrB,UAAU,EACV,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,GAAG,KAAK,CAAC,CACzC,CAAC;YACF,MAAM,SAAS,GAAG,KAAK,IAAI,CAAC,CAAC;YAC7B,MAAM,QAAQ,GAAG,MAAM,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;YACxC,MAAM,CAAC,SAAS,CAAC;gBACf,KAAK,GAAG,CAAC,KAAK,CAAC;oBACb,CAAC,CAAC,CAAC,MAAM,IAAI,CAAC,CAAC,GAAG,CAAC,QAAQ,GAAG,GAAG,CAAC;oBAClC,CAAC,CAAC,CAAC,QAAQ,GAAG,IAAI,CAAC,GAAG,MAAM,CAAC;QACnC,CAAC;QAED,OAAO,IAAI,cAAc,CAAC,MAAM,EAAE,WAAW,CAAC,MAAM,CAAC,CAAC;IACxD,CAAC;IAED;;OAEG;IACH,MAAM,CAAC,IAAI,CAAC,MAAkB,EAAE,IAAY;QAC1C,OAAO,IAAI,cAAc,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;IAC1C,CAAC;IAEQ,OAAO,CAAa;IACpB,KAAK,CAAS;IAEvB;;;OAGG;IACH,YAAoB,MAAkB,EAAE,IAAY;QAClD,IAAI,CAAC,OAAO,GAAG,MAAM,CAAC;QACtB,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC;IACpB,CAAC;IAED;;OAEG;IACH,IAAI,IAAI;QACN,OAAO,IAAI,CAAC,KAAK,CAAC;IACpB,CAAC;IAED;;;;;OAKG;IACH,QAAQ,CAAC,KAAa;QACpB,IAAI,KAAK,GAAG,CAAC,IAAI,KAAK,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC;YACrC,OAAO,CAAC,CAAC;QACX,CAAC;QACD,MAAM,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,KAAK,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC;QAC3C,OAAO,KAAK,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,GAAG,GAAG,CAAC;IAClD,CAAC;IAED;;;;;;;;OAQG;IACH,MAAM,CAAC,KAAa;QAClB,OAAO,UAAU,GAAG,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,GAAG,WAAW,CAAC;IACzD,CAAC;IAED;;OAEG;IACH,SAAS;QACP,OAAO,IAAI,CAAC,OAAO,CAAC;IACtB,CAAC;CACF"}
1
+ {"version":3,"file":"frequency-table.js","sourceRoot":"","sources":["../../src/dictionary/frequency-table.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AACH,MAAM,CAAC,MAAM,iBAAiB,GAAG,EAAE,CAAC;AAEpC,MAAM,UAAU,GAAG,iBAAiB,GAAG,CAAC,CAAC;AAEzC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAoCG;AACH,MAAM,CAAC,MAAM,WAAW,GAAG,IAAI,CAAC;AAEhC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA0CG;AACH,MAAM,CAAC,MAAM,cAAc,GAAG,CAAC,CAAC;AAEhC;;;;;;;;GAQG;AACH,MAAM,UAAU,kBAAkB,CAChC,SAAiB,EACjB,YAAqB;IAErB,OAAO,YAAY,IAAI,SAAS,IAAI,CAAC,CAAC,CAAC,CAAC,cAAc,CAAC,CAAC,CAAC,SAAS,CAAC;AACrE,CAAC;AAED;;;;;;GAMG;AACH,MAAM,UAAU,iBAAiB,CAAC,KAAa,EAAE,OAAe;IAC9D,OAAO,UAAU,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC;AAClE,CAAC;AAED;;;;;;GAMG;AACH,MAAM,OAAO,cAAc;IACzB;;;;;;;;;;;OAWG;IACH,MAAM,CAAC,KAAK,CAAC,WAA8B;QACzC,qEAAqE;QACrE,4EAA4E;QAC5E,0EAA0E;QAC1E,IAAI,OAAO,GAAG,CAAC,CAAC;QAChB,KAAK,MAAM,SAAS,IAAI,WAAW,EAAE,CAAC;YACpC,IAAI,SAAS,GAAG,OAAO,EAAE,CAAC;gBACxB,OAAO,GAAG,SAAS,CAAC;YACtB,CAAC;QACH,CAAC;QACD,MAAM,KAAK,GAAG,OAAO,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,UAAU,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;QAEnE,MAAM,MAAM,GAAG,IAAI,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,WAAW,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC;QACjE,KAAK,MAAM,CAAC,KAAK,EAAE,SAAS,CAAC,IAAI,WAAW,CAAC,OAAO,EAAE,EAAE,CAAC;YACvD,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,SAAS,CAAC,CAAC;YACxC,MAAM,MAAM,GAAG,IAAI,CAAC,GAAG,CACrB,UAAU,EACV,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,GAAG,KAAK,CAAC,CACzC,CAAC;YACF,MAAM,SAAS,GAAG,KAAK,IAAI,CAAC,CAAC;YAC7B,MAAM,QAAQ,GAAG,MAAM,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;YACxC,MAAM,CAAC,SAAS,CAAC;gBACf,KAAK,GAAG,CAAC,KAAK,CAAC;oBACb,CAAC,CAAC,CAAC,MAAM,IAAI,CAAC,CAAC,GAAG,CAAC,QAAQ,GAAG,GAAG,CAAC;oBAClC,CAAC,CAAC,CAAC,QAAQ,GAAG,IAAI,CAAC,GAAG,MAAM,CAAC;QACnC,CAAC;QAED,OAAO,IAAI,cAAc,CAAC,MAAM,EAAE,WAAW,CAAC,MAAM,CAAC,CAAC;IACxD,CAAC;IAED;;OAEG;IACH,MAAM,CAAC,IAAI,CAAC,MAAkB,EAAE,IAAY;QAC1C,OAAO,IAAI,cAAc,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;IAC1C,CAAC;IAEQ,OAAO,CAAa;IACpB,KAAK,CAAS;IAEvB;;;OAGG;IACH,YAAoB,MAAkB,EAAE,IAAY;QAClD,IAAI,CAAC,OAAO,GAAG,MAAM,CAAC;QACtB,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC;IACpB,CAAC;IAED;;OAEG;IACH,IAAI,IAAI;QACN,OAAO,IAAI,CAAC,KAAK,CAAC;IACpB,CAAC;IAED;;;;;OAKG;IACH,QAAQ,CAAC,KAAa;QACpB,IAAI,KAAK,GAAG,CAAC,IAAI,KAAK,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC;YACrC,OAAO,CAAC,CAAC;QACX,CAAC;QACD,MAAM,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,KAAK,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC;QAC3C,OAAO,KAAK,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,GAAG,GAAG,CAAC;IAClD,CAAC;IAED;;;;;;;;OAQG;IACH,MAAM,CAAC,KAAa;QAClB,OAAO,UAAU,GAAG,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,GAAG,WAAW,CAAC;IACzD,CAAC;IAED;;OAEG;IACH,SAAS;QACP,OAAO,IAAI,CAAC,OAAO,CAAC;IACtB,CAAC;CACF"}
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@kensio/pinyinjs",
3
- "version": "1.7.9",
3
+ "version": "1.7.10",
4
4
  "description": "Chinese hanzi to pinyin conversion",
5
5
  "repository": "https://github.com/KensioSoftware/pinyinjs",
6
6
  "homepage": "https://pinyinjs.dev",
@@ -69,7 +69,7 @@
69
69
  "conventional-changelog-conventionalcommits": "9.3.1",
70
70
  "fta-cli": "^3.0.0",
71
71
  "jiti": "^2.7.0",
72
- "oxfmt": "^0.63.0",
72
+ "oxfmt": "^0.64.0",
73
73
  "oxlint": "^1.77.0",
74
74
  "oxlint-tsgolint": "^7.0.2001",
75
75
  "semantic-release": "25.0.9",