@ohos-ports/nodejieba 0.0.0-stage → 3.5.8-beta.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +360 -0
- package/LICENSE +21 -0
- package/README.md +270 -2
- package/binding.gyp +63 -0
- package/build/Release/nodejieba.node +0 -0
- package/index.js +77 -0
- package/lib/index.cpp +3 -0
- package/lib/nodejieba.cpp +218 -0
- package/lib/nodejieba.h +28 -0
- package/lib/utils.h +47 -0
- package/package.json +62 -4
- package/submodules/cppjieba/CHANGELOG.md +305 -0
- package/submodules/cppjieba/deps/limonp/CHANGELOG.md +160 -0
- package/submodules/cppjieba/deps/limonp/include/limonp/ArgvContext.hpp +70 -0
- package/submodules/cppjieba/deps/limonp/include/limonp/Closure.hpp +206 -0
- package/submodules/cppjieba/deps/limonp/include/limonp/Colors.hpp +31 -0
- package/submodules/cppjieba/deps/limonp/include/limonp/Condition.hpp +38 -0
- package/submodules/cppjieba/deps/limonp/include/limonp/Config.hpp +103 -0
- package/submodules/cppjieba/deps/limonp/include/limonp/ForcePublic.hpp +7 -0
- package/submodules/cppjieba/deps/limonp/include/limonp/LocalVector.hpp +139 -0
- package/submodules/cppjieba/deps/limonp/include/limonp/Logging.hpp +90 -0
- package/submodules/cppjieba/deps/limonp/include/limonp/NonCopyable.hpp +21 -0
- package/submodules/cppjieba/deps/limonp/include/limonp/StdExtension.hpp +157 -0
- package/submodules/cppjieba/deps/limonp/include/limonp/StringUtil.hpp +386 -0
- package/submodules/cppjieba/dict/README.md +31 -0
- package/submodules/cppjieba/dict/hmm_model.utf8 +34 -0
- package/submodules/cppjieba/dict/idf.utf8 +258826 -0
- package/submodules/cppjieba/dict/jieba.dict.utf8 +348982 -0
- package/submodules/cppjieba/dict/pos_dict/char_state_tab.utf8 +6653 -0
- package/submodules/cppjieba/dict/pos_dict/prob_emit.utf8 +166 -0
- package/submodules/cppjieba/dict/pos_dict/prob_start.utf8 +259 -0
- package/submodules/cppjieba/dict/pos_dict/prob_trans.utf8 +5222 -0
- package/submodules/cppjieba/dict/stop_words.utf8 +1534 -0
- package/submodules/cppjieba/dict/user.dict.utf8 +4 -0
- package/submodules/cppjieba/include/cppjieba/DictTrie.hpp +285 -0
- package/submodules/cppjieba/include/cppjieba/FullSegment.hpp +93 -0
- package/submodules/cppjieba/include/cppjieba/HMMModel.hpp +129 -0
- package/submodules/cppjieba/include/cppjieba/HMMSegment.hpp +190 -0
- package/submodules/cppjieba/include/cppjieba/Jieba.hpp +169 -0
- package/submodules/cppjieba/include/cppjieba/KeywordExtractor.hpp +153 -0
- package/submodules/cppjieba/include/cppjieba/MPSegment.hpp +137 -0
- package/submodules/cppjieba/include/cppjieba/MixSegment.hpp +109 -0
- package/submodules/cppjieba/include/cppjieba/PosTagger.hpp +77 -0
- package/submodules/cppjieba/include/cppjieba/PreFilter.hpp +54 -0
- package/submodules/cppjieba/include/cppjieba/QuerySegment.hpp +89 -0
- package/submodules/cppjieba/include/cppjieba/SegmentBase.hpp +46 -0
- package/submodules/cppjieba/include/cppjieba/SegmentTagged.hpp +23 -0
- package/submodules/cppjieba/include/cppjieba/TextRankExtractor.hpp +190 -0
- package/submodules/cppjieba/include/cppjieba/Trie.hpp +200 -0
- package/submodules/cppjieba/include/cppjieba/Unicode.hpp +227 -0
- package/test/load_dict_test.js +14 -0
- package/test/missing_binding_test.js +42 -0
- package/test/test.js +366 -0
- package/test/testdata/userdict.utf8 +1 -0
- package/types/index.d.ts +30 -0
package/CHANGELOG.md
ADDED
|
@@ -0,0 +1,360 @@
|
|
|
1
|
+
# CHANGELOG
|
|
2
|
+
|
|
3
|
+
## v3.4.3
|
|
4
|
+
|
|
5
|
+
+ 8723bbb [submodule/cppjieba] from v5.1.3 to v5.4.0
|
|
6
|
+
+ 7787450 Update stale-issues.yml: 3y -> 1y
|
|
7
|
+
+ 90d5b6b [github/actions] stale 5 years -> 3 years
|
|
8
|
+
+ b69d1fe [github/actions] stale issues
|
|
9
|
+
+ 5241ef1 [.github][npm publish] upgrade */v2 to */v4 and node-version 18->20
|
|
10
|
+
|
|
11
|
+
## v3.4.2
|
|
12
|
+
|
|
13
|
+
+ 9bc3979 [package] clean unused deps
|
|
14
|
+
+ e8988c5 git rm .travis.yml
|
|
15
|
+
+ 839f203 Merge pull request #236 from yanyiwu/dependabot/npm_and_yarn/js-yaml-3.14.1
|
|
16
|
+
+ 26f0872 Bump js-yaml from 3.6.1 to 3.14.1
|
|
17
|
+
+ 1788029 Merge pull request #235 from yanyiwu/dependabot/npm_and_yarn/multi-037252cf8f
|
|
18
|
+
+ 641b2d7 Bump minimist, coveralls and mocha
|
|
19
|
+
+ e01e1c4 [CI] GitHub Release: publish --release instead of draft
|
|
20
|
+
|
|
21
|
+
## v3.4.1
|
|
22
|
+
|
|
23
|
+
+ [demo] using nodejieba-demo instead of test/demo.js
|
|
24
|
+
+ git submodule update --remote to upgrade cppjieba from v5.1.2 to v5.1.3
|
|
25
|
+
|
|
26
|
+
## v3.4.0
|
|
27
|
+
|
|
28
|
+
+ package-lock.json updated by 'npm install'
|
|
29
|
+
+ npm outdated; npm update;
|
|
30
|
+
|
|
31
|
+
## v3.3.1
|
|
32
|
+
|
|
33
|
+
+ [CI] fix github/actions:auto-release
|
|
34
|
+
|
|
35
|
+
## v3.3.0
|
|
36
|
+
|
|
37
|
+
+ [CI] auto github/release by node-pre-gyp-github
|
|
38
|
+
+ [github/actions] CI add windows,macos, node-22, removed node-16
|
|
39
|
+
|
|
40
|
+
## v3.2.0
|
|
41
|
+
|
|
42
|
+
+ [submodule] rm dict and deps, use submodules/cppjieba instead
|
|
43
|
+
|
|
44
|
+
## v3.1.1
|
|
45
|
+
|
|
46
|
+
+ Using GitHub Actions Workflow to Publish NPM Package
|
|
47
|
+
|
|
48
|
+
## v3.1.0
|
|
49
|
+
|
|
50
|
+
+ Merged [pull request 220](https://github.com/yanyiwu/nodejieba/pull/220)
|
|
51
|
+
+ Merged [pull request 212](https://github.com/yanyiwu/nodejieba/pull/212)
|
|
52
|
+
+ Merged [pull request 210](https://github.com/yanyiwu/nodejieba/pull/210)
|
|
53
|
+
+ Merged [pull request 207](https://github.com/yanyiwu/nodejieba/pull/207)
|
|
54
|
+
+ Merged [pull request 206](https://github.com/yanyiwu/nodejieba/pull/206)
|
|
55
|
+
+ Merged [pull request 204](https://github.com/yanyiwu/nodejieba/pull/204)
|
|
56
|
+
|
|
57
|
+
## v2.6.0
|
|
58
|
+
|
|
59
|
+
+ Merged [pull request 202](https://github.com/yanyiwu/nodejieba/pull/202)
|
|
60
|
+
|
|
61
|
+
## v2.5.2
|
|
62
|
+
|
|
63
|
+
+ Merged [pull request 193](https://github.com/yanyiwu/nodejieba/pull/193)
|
|
64
|
+
|
|
65
|
+
## v2.5.1
|
|
66
|
+
|
|
67
|
+
+ Merged [pull request 179](https://github.com/yanyiwu/nodejieba/pull/179)
|
|
68
|
+
+ Since v2.5.0, [node-addon-api](https://github.com/nodejs/node-addon-api) is used, and only Node.js versions 10.0.0 and above are supported. There was an error in the previous description in the v2.5.0 version.
|
|
69
|
+
|
|
70
|
+
## v2.5.0
|
|
71
|
+
|
|
72
|
+
+ merged [pull request 177](https://github.com/yanyiwu/nodejieba/pull/177)
|
|
73
|
+
+ only supports Node.js 7.0.0 and above.
|
|
74
|
+
|
|
75
|
+
## v2.4.2
|
|
76
|
+
|
|
77
|
+
+ merged [pull request 173](https://github.com/yanyiwu/nodejieba/pull/173)
|
|
78
|
+
|
|
79
|
+
## v2.4.1
|
|
80
|
+
|
|
81
|
+
+ upgrade deps/limonp -> v0.6.3
|
|
82
|
+
+ upgrade deps/cppjieba -> v5.0.3
|
|
83
|
+
+ merged [pull request 148](https://github.com/yanyiwu/nodejieba/pull/148)
|
|
84
|
+
|
|
85
|
+
## v2.4.0
|
|
86
|
+
|
|
87
|
+
+ revert commit-6600629 to be compatiable with cppjieba.
|
|
88
|
+
+ upgrade deps/limonp -> v0.6.2
|
|
89
|
+
+ upgrade deps/cppjieba -> v5.0.2
|
|
90
|
+
|
|
91
|
+
## v2.3.5
|
|
92
|
+
|
|
93
|
+
+ merged [pull request 143](https://github.com/yanyiwu/nodejieba/pull/143)
|
|
94
|
+
+ add release deployment in travis
|
|
95
|
+
|
|
96
|
+
## v2.3.3
|
|
97
|
+
|
|
98
|
+
+ merged [pull request 142](https://github.com/yanyiwu/nodejieba/pull/142)
|
|
99
|
+
|
|
100
|
+
## v2.3.2
|
|
101
|
+
|
|
102
|
+
+ merged [pull request 140](https://github.com/yanyiwu/nodejieba/pull/140)
|
|
103
|
+
|
|
104
|
+
## v2.3.1
|
|
105
|
+
|
|
106
|
+
+ merged [pull request 132](https://github.com/yanyiwu/nodejieba/pull/132)
|
|
107
|
+
|
|
108
|
+
## v2.3.0
|
|
109
|
+
|
|
110
|
+
+ merged [pull request 125](https://github.com/yanyiwu/nodejieba/pull/125)
|
|
111
|
+
|
|
112
|
+
## v2.2.7
|
|
113
|
+
|
|
114
|
+
+ support typescript [pull request 118](https://github.com/yanyiwu/nodejieba/pull/118)
|
|
115
|
+
|
|
116
|
+
## v2.2.6
|
|
117
|
+
|
|
118
|
+
+ fix bug [pull request 113](https://github.com/yanyiwu/nodejieba/pull/113)
|
|
119
|
+
|
|
120
|
+
## v2.2.5
|
|
121
|
+
|
|
122
|
+
+ fix bug [pull request 89](https://github.com/yanyiwu/nodejieba/pull/89)
|
|
123
|
+
|
|
124
|
+
## v2.2.4
|
|
125
|
+
|
|
126
|
+
+ support "tag" in insertWord function. [pull request 71](https://github.com/yanyiwu/nodejieba/pull/71)
|
|
127
|
+
|
|
128
|
+
## v2.2.3
|
|
129
|
+
|
|
130
|
+
+ update cppjieba -> 4a755dff to try fixing [issue 65](https://github.com/yanyiwu/nodejieba/issues/65)
|
|
131
|
+
|
|
132
|
+
## v2.2.2
|
|
133
|
+
|
|
134
|
+
+ upgrade cppjieba -> v4.8.1
|
|
135
|
+
+ (word freq tag) format in user dict has been supported.
|
|
136
|
+
|
|
137
|
+
## v2.2.1
|
|
138
|
+
|
|
139
|
+
+ fix bug in v2.2.0 about 'cutSmall'
|
|
140
|
+
|
|
141
|
+
## v2.2.0
|
|
142
|
+
|
|
143
|
+
+ add api `cutSmall` to cut words in specific word length limit
|
|
144
|
+
|
|
145
|
+
## v2.1.0
|
|
146
|
+
|
|
147
|
+
+ function `tag,extract` return objects, instead of strings.
|
|
148
|
+
+ fix issue [issue 58](https://github.com/yanyiwu/nodejieba/issues/58)
|
|
149
|
+
|
|
150
|
+
## v2.0.0
|
|
151
|
+
|
|
152
|
+
+ Upgrade cppjieba -> v4.8.0
|
|
153
|
+
+ rewrite `cut` function:
|
|
154
|
+
+ add `cutAll` for old `cut(s, "FULL")` function.
|
|
155
|
+
+ add `cutHMM` for old `cut(s, "HMM")` function.
|
|
156
|
+
+ add `cutForSearch` for old `cut(s, "Query")` function.
|
|
157
|
+
|
|
158
|
+
## v1.4.11
|
|
159
|
+
|
|
160
|
+
+ Upgrade dependencies
|
|
161
|
+
|
|
162
|
+
+ "nan": "~2.2.0" -> "~2.3.0"
|
|
163
|
+
|
|
164
|
+
+ Upgrade devDependencies
|
|
165
|
+
|
|
166
|
+
+ "mocha": "~2.3.4" -> "~2.4.5"
|
|
167
|
+
+ "should": "~8.0.2" -> "~8.3.1"
|
|
168
|
+
|
|
169
|
+
+ Upgrade deps/
|
|
170
|
+
|
|
171
|
+
+ cppjieba -> v4.7.0
|
|
172
|
+
+ limonp -> v0.6.1
|
|
173
|
+
|
|
174
|
+
## v1.4.10
|
|
175
|
+
|
|
176
|
+
Upgrade to fix potential trouble:
|
|
177
|
+
|
|
178
|
+
+ limonp -> v0.5.6
|
|
179
|
+
+ cppjieba -> v4.5.2
|
|
180
|
+
|
|
181
|
+
## v1.4.9
|
|
182
|
+
|
|
183
|
+
Upgrade limonp and cppjieba to fix [issue48](https://github.com/yanyiwu/nodejieba/issues/48)
|
|
184
|
+
|
|
185
|
+
* limonp -> v0.5.4
|
|
186
|
+
* cppjieba -> v4.5.0
|
|
187
|
+
|
|
188
|
+
## v1.4.8
|
|
189
|
+
|
|
190
|
+
Upgrade:
|
|
191
|
+
|
|
192
|
+
* cppjieba -> v4.4.1 to fix bug, see details in [CppJieba ChangeLog v4.4.1](https://github.com/yanyiwu/cppjieba/blob/master/ChangeLog.md#v441)
|
|
193
|
+
|
|
194
|
+
## v1.4.7
|
|
195
|
+
|
|
196
|
+
Upgrade:
|
|
197
|
+
|
|
198
|
+
* limonp -> v0.5.3, fix incompatibility problem in Windows.
|
|
199
|
+
|
|
200
|
+
## v1.4.6
|
|
201
|
+
|
|
202
|
+
Change source codes layout
|
|
203
|
+
|
|
204
|
+
* lib/jieba/limonp -> deps/limonp
|
|
205
|
+
* lib/jieba -> deps/cppjieba
|
|
206
|
+
|
|
207
|
+
Upgrade:
|
|
208
|
+
|
|
209
|
+
* limonp -> v0.5.2, fix incompatibility problem of [issue45](https://github.com/yanyiwu/nodejieba/issues/45)
|
|
210
|
+
* cppjieba -> v4.3.2
|
|
211
|
+
|
|
212
|
+
## v1.4.5
|
|
213
|
+
|
|
214
|
+
* 修复多余日志输出的问题。
|
|
215
|
+
|
|
216
|
+
## v1.4.4
|
|
217
|
+
|
|
218
|
+
* 升级 [cppjieba] 到最新版本 v4.2.1
|
|
219
|
+
|
|
220
|
+
## v1.4.3
|
|
221
|
+
|
|
222
|
+
* 升级 [cppjieba] 到最新版本 v4.1.2
|
|
223
|
+
|
|
224
|
+
## v1.4.2
|
|
225
|
+
|
|
226
|
+
* 升级 package.json 里面的依赖库。
|
|
227
|
+
|
|
228
|
+
## v1.4.1
|
|
229
|
+
|
|
230
|
+
* [enhancement issue]: https://github.com/yanyiwu/nodejieba/issues/39
|
|
231
|
+
|
|
232
|
+
## v1.4.0
|
|
233
|
+
|
|
234
|
+
* 对MP算法的分词方式新增一个粒度参数,解决[issue29]关于细粒度分词的问题。
|
|
235
|
+
|
|
236
|
+
## v1.3.0
|
|
237
|
+
|
|
238
|
+
* 合并 [pr37], 修复 [issue36] 。 放弃对旧版本 `node.js v0.11` 的支持。
|
|
239
|
+
|
|
240
|
+
## v1.2.2
|
|
241
|
+
|
|
242
|
+
* 修复依赖的nan版本的问题。
|
|
243
|
+
|
|
244
|
+
## v1.2.1
|
|
245
|
+
|
|
246
|
+
* 修复load函数重复调用时内存没有释放的问题。
|
|
247
|
+
|
|
248
|
+
## v1.2.0
|
|
249
|
+
|
|
250
|
+
* 同时支持【不配置词典文件直接使用】和【灵活配置词典】的需求。
|
|
251
|
+
|
|
252
|
+
## v1.1.0
|
|
253
|
+
|
|
254
|
+
* 新增支持动态增加词库的API函数: `insertWord`
|
|
255
|
+
|
|
256
|
+
## v1.0.3
|
|
257
|
+
|
|
258
|
+
* Merge Pull Request [pr28],显示声明依赖的是 node 后缀的文件,解决一些库重写了 require 的问题。
|
|
259
|
+
|
|
260
|
+
## v1.0.2
|
|
261
|
+
|
|
262
|
+
* 修复在 windows 上编译不通过的问题,提升兼容性。
|
|
263
|
+
|
|
264
|
+
## v1.0.1
|
|
265
|
+
|
|
266
|
+
* 修复在iojs v2.2.1 上编译不通过的问题,提升兼容性。
|
|
267
|
+
|
|
268
|
+
## v1.0.0
|
|
269
|
+
|
|
270
|
+
* 使用[cppjieba] v3.0.0 版本,使用更简洁的接口函数,可以使用参数指定切词方法。
|
|
271
|
+
* `require("nodejieba")` 时自动载入词典,用户无需再烦心字典的事情,实现即插即用。
|
|
272
|
+
* 听从 [issue23] 的建议,废除异步调用的接口,现在的接口函数都是同步调用函数,分别是 `cut`, `tag`, `extract` 。因为分词毕竟是CPU密集型的事情,没必要使用异步调用。
|
|
273
|
+
|
|
274
|
+
## v0.2.13
|
|
275
|
+
|
|
276
|
+
* 升级[nan]到 `^v1.8.0` 以支持 `iojs-v2.x`
|
|
277
|
+
|
|
278
|
+
## v0.2.12
|
|
279
|
+
|
|
280
|
+
* 新增以下默认词典地址,让使用者不需要自己写词典的相对地址。
|
|
281
|
+
* nodejieba.DEFAULT_DICT
|
|
282
|
+
* nodejieba.DEFAULT_HMM_DICT
|
|
283
|
+
* nodejieba.DEFAULT_USER_DICT
|
|
284
|
+
* nodejieba.DEFAULT_IDF_DICT
|
|
285
|
+
* nodejieba.DEFAULT_STOP_WORD_DICT
|
|
286
|
+
|
|
287
|
+
## v0.2.11
|
|
288
|
+
|
|
289
|
+
* 更新 [cppjieba] 。
|
|
290
|
+
* 打印更详细的初始化错误日志信息。
|
|
291
|
+
|
|
292
|
+
## v0.2.10
|
|
293
|
+
|
|
294
|
+
* 避免一些在Mac系统下的词典载入日志,更干净的用户体验。
|
|
295
|
+
|
|
296
|
+
## v0.2.9
|
|
297
|
+
|
|
298
|
+
* 支持vs2012编译。
|
|
299
|
+
|
|
300
|
+
## v0.2.8(deleted)
|
|
301
|
+
|
|
302
|
+
* 支持vs2012编译。
|
|
303
|
+
|
|
304
|
+
## v0.2.7
|
|
305
|
+
|
|
306
|
+
* 关键词抽取支持自定义词典(可选参数)。
|
|
307
|
+
|
|
308
|
+
## v0.2.6
|
|
309
|
+
|
|
310
|
+
* 修复不同Node版本的兼容性问题,在v0.11.13下测试通过。
|
|
311
|
+
* 支持自定义词典(可选参数)。
|
|
312
|
+
* 增加关键词抽取功能。
|
|
313
|
+
|
|
314
|
+
## v0.2.5
|
|
315
|
+
|
|
316
|
+
* 增加词性标注功能
|
|
317
|
+
|
|
318
|
+
## v0.2.4
|
|
319
|
+
|
|
320
|
+
* 更新 package 兼容更低版本的 npm
|
|
321
|
+
|
|
322
|
+
## v0.2.3
|
|
323
|
+
|
|
324
|
+
* 更新 cppjieba ,减少内存使用。
|
|
325
|
+
|
|
326
|
+
## v0.2.2
|
|
327
|
+
|
|
328
|
+
* 在queryLoadDict 函数中增加query模式的粒度阈值作为可选参数。
|
|
329
|
+
|
|
330
|
+
## v0.2.1
|
|
331
|
+
|
|
332
|
+
* 增加搜索引擎分词模式,分别对应的调用函数是 `queryLoadDict, queryCutSync, queryCut`。
|
|
333
|
+
|
|
334
|
+
## v0.2.0
|
|
335
|
+
|
|
336
|
+
* 将原来的 cut 阻塞分词模式改为非阻塞模式
|
|
337
|
+
* 阻塞分词模型的函数名为 cutSync
|
|
338
|
+
|
|
339
|
+
## v0.1.4
|
|
340
|
+
|
|
341
|
+
* 修复关于较低版本编译器需要使用`tr1/unordered_map`导致和`node-gyp`编译选项`-fno-rtti`冲突的编译错误问题。
|
|
342
|
+
|
|
343
|
+
## v0.1.3
|
|
344
|
+
|
|
345
|
+
* 更新[cppjieba],支持更低版本的g++。
|
|
346
|
+
|
|
347
|
+
## v0.1.2
|
|
348
|
+
|
|
349
|
+
* 更新[cppjieba],使用`less_memory`这个branch来减少Trie树内存的开销。
|
|
350
|
+
|
|
351
|
+
## v0.1.1
|
|
352
|
+
|
|
353
|
+
* 依照node的c++扩展的常规写法,对cppjieba进行简单的包装,并已`npm publish`
|
|
354
|
+
|
|
355
|
+
[cppjieba]:http://github.com/yanyiwu/cppjieba.git
|
|
356
|
+
[nan]:https://github.com/nodejs/nan/
|
|
357
|
+
[issue23]:https://github.com/yanyiwu/nodejieba/issues/23
|
|
358
|
+
[pr28]:https://github.com/yanyiwu/nodejieba/pull/28
|
|
359
|
+
[issue36]:https://github.com/yanyiwu/nodejieba/issues/36
|
|
360
|
+
[issue29]:https://github.com/yanyiwu/nodejieba/issues/29
|
package/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2021 Yanyi Wu
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
package/README.md
CHANGED
|
@@ -1,3 +1,271 @@
|
|
|
1
|
-
|
|
1
|
+
[](https://github.com/yanyiwu/nodejieba/actions/workflows/test.yml)
|
|
2
|
+
[](https://opencollective.com/nodejieba) [](https://github.com/yanyiwu/)
|
|
3
|
+
[](https://github.com/yanyiwu/nodejieba)
|
|
4
|
+
[](https://github.com/yanyiwu/blog/blob/posts2023archive/_posts/2015-06-14-jieba-series-performance-test.md)
|
|
5
|
+
[](http://yanyiwu.mit-license.org)
|
|
6
|
+
[](https://www.npmjs.org/package/nodejieba)
|
|
7
|
+
[](https://www.npmjs.org/package/nodejieba)
|
|
8
|
+
[](https://codeclimate.com/github/yanyiwu/nodejieba)
|
|
2
9
|
|
|
3
|
-
|
|
10
|
+
- - -
|
|
11
|
+
|
|
12
|
+
# NodeJieba "结巴"分词的Node.js版本
|
|
13
|
+
|
|
14
|
+
## 介绍
|
|
15
|
+
|
|
16
|
+
`NodeJieba`是"结巴"中文分词的 Node.js 版本实现,
|
|
17
|
+
由[CppJieba]提供底层分词算法实现,
|
|
18
|
+
是兼具高性能和易用性两者的 Node.js 中文分词组件。
|
|
19
|
+
|
|
20
|
+
## 特点
|
|
21
|
+
|
|
22
|
+
+ 词典载入方式灵活,无需配置词典路径也可使用,需要定制自己的词典路径时也可灵活定制。
|
|
23
|
+
+ 底层算法实现是C++,性能高效。
|
|
24
|
+
+ 支持多种分词算法,各种分词算法见[CppJieba]的README.md介绍。
|
|
25
|
+
+ 支持动态补充词库。
|
|
26
|
+
+ 支持TypeScript,提供完整的类型定义。
|
|
27
|
+
|
|
28
|
+
对实现细节感兴趣的请看如下博文:
|
|
29
|
+
|
|
30
|
+
+ [Node.js的C++扩展初体验之NodeJieba]
|
|
31
|
+
+ [由NodeJieba谈谈Node.js异步实现]
|
|
32
|
+
|
|
33
|
+
## 安装
|
|
34
|
+
|
|
35
|
+
```sh
|
|
36
|
+
npm install nodejieba
|
|
37
|
+
```
|
|
38
|
+
|
|
39
|
+
## 快速开始
|
|
40
|
+
|
|
41
|
+
```js
|
|
42
|
+
var nodejieba = require("nodejieba");
|
|
43
|
+
var result = nodejieba.cut("南京市长江大桥");
|
|
44
|
+
console.log(result);
|
|
45
|
+
//["南京市","长江大桥"]
|
|
46
|
+
```
|
|
47
|
+
|
|
48
|
+
更多示例请参考 [demo](https://github.com/yanyiwu/nodejieba-demo)
|
|
49
|
+
|
|
50
|
+
### 词典载入可灵活配置
|
|
51
|
+
|
|
52
|
+
如果没有主动调用词典函数时,
|
|
53
|
+
则会在第一次调用cut等功能函数时,自动载入默认词典。
|
|
54
|
+
|
|
55
|
+
如果要主动触发词典载入,则使用以下函数主动触发。
|
|
56
|
+
|
|
57
|
+
```js
|
|
58
|
+
nodejieba.load();
|
|
59
|
+
```
|
|
60
|
+
|
|
61
|
+
以上用法会自动载入所有默认词典,
|
|
62
|
+
如果需要载入自己的词典,而不是默认词典。
|
|
63
|
+
比如想要载入自己的用户词典,则使用以下函数:
|
|
64
|
+
|
|
65
|
+
```js
|
|
66
|
+
nodejieba.load({
|
|
67
|
+
userDict: './test/testdata/userdict.utf8',
|
|
68
|
+
});
|
|
69
|
+
```
|
|
70
|
+
|
|
71
|
+
字典载入函数load的参数项都是可选的,
|
|
72
|
+
如果没有对应的项则自动填充默认参数。
|
|
73
|
+
所以上面这段代码和下面这代代码是等价的。
|
|
74
|
+
|
|
75
|
+
```js
|
|
76
|
+
nodejieba.load({
|
|
77
|
+
dict: nodejieba.DEFAULT_DICT,
|
|
78
|
+
hmmDict: nodejieba.DEFAULT_HMM_DICT,
|
|
79
|
+
userDict: './test/testdata/userdict.utf8',
|
|
80
|
+
idfDict: nodejieba.DEFAULT_IDF_DICT,
|
|
81
|
+
stopWordDict: nodejieba.DEFAULT_STOP_WORD_DICT,
|
|
82
|
+
});
|
|
83
|
+
```
|
|
84
|
+
|
|
85
|
+
#### 词典说明
|
|
86
|
+
|
|
87
|
+
+ dict: 主词典,带权重和词性标签,建议使用默认词典。
|
|
88
|
+
+ hmmDict: 隐式马尔科夫模型,建议使用默认词典。
|
|
89
|
+
+ userDict: 用户词典,建议自己根据需要定制。
|
|
90
|
+
+ idfDict: 关键词抽取所需的idf信息。
|
|
91
|
+
+ stopWordDict: 关键词抽取所需的停用词列表。
|
|
92
|
+
|
|
93
|
+
## API 文档
|
|
94
|
+
|
|
95
|
+
### 分词
|
|
96
|
+
|
|
97
|
+
#### 1. 默认分词
|
|
98
|
+
|
|
99
|
+
```js
|
|
100
|
+
var nodejieba = require("nodejieba");
|
|
101
|
+
var result = nodejieba.cut("南京市长江大桥");
|
|
102
|
+
console.log(result);
|
|
103
|
+
// ["南京市", "长江大桥"]
|
|
104
|
+
```
|
|
105
|
+
|
|
106
|
+
#### 2. 使用HMM模型分词
|
|
107
|
+
|
|
108
|
+
```js
|
|
109
|
+
var result = nodejieba.cutHMM("南京市长江大桥");
|
|
110
|
+
console.log(result);
|
|
111
|
+
// ["南京市", "长江大桥"]
|
|
112
|
+
```
|
|
113
|
+
|
|
114
|
+
#### 3. 全模式分词
|
|
115
|
+
|
|
116
|
+
```js
|
|
117
|
+
var result = nodejieba.cutAll("南京市长江大桥");
|
|
118
|
+
console.log(result);
|
|
119
|
+
// ["南京", "南京市", "市长", "长江", "长江大桥", "大桥"]
|
|
120
|
+
```
|
|
121
|
+
|
|
122
|
+
#### 4. 搜索引擎模式分词
|
|
123
|
+
|
|
124
|
+
```js
|
|
125
|
+
var result = nodejieba.cutForSearch("南京市长江大桥");
|
|
126
|
+
console.log(result);
|
|
127
|
+
// ["南京", "市", "长江", "大桥", "南京市", "长江大桥"]
|
|
128
|
+
```
|
|
129
|
+
|
|
130
|
+
#### 5. 小粒度分词
|
|
131
|
+
|
|
132
|
+
```js
|
|
133
|
+
var result = nodejieba.cutSmall("南京市长江大桥", 3);
|
|
134
|
+
console.log(result);
|
|
135
|
+
// ["南京市", "长江大桥"]
|
|
136
|
+
```
|
|
137
|
+
|
|
138
|
+
### 词性标注
|
|
139
|
+
|
|
140
|
+
```js
|
|
141
|
+
var nodejieba = require("nodejieba");
|
|
142
|
+
var result = nodejieba.tag("红掌拨清波");
|
|
143
|
+
console.log(result);
|
|
144
|
+
// [ { word: '红掌', tag: 'n' },
|
|
145
|
+
// { word: '拨', tag: 'v' },
|
|
146
|
+
// { word: '清波', tag: 'n' } ]
|
|
147
|
+
```
|
|
148
|
+
|
|
149
|
+
### 关键词提取
|
|
150
|
+
|
|
151
|
+
```js
|
|
152
|
+
var nodejieba = require("nodejieba");
|
|
153
|
+
var sentence = "我是拖拉机学院手扶拖拉机专业的。不用多久,我就会升职加薪,当上CEO,走上人生巅峰。";
|
|
154
|
+
var result = nodejieba.extract(sentence, 5);
|
|
155
|
+
console.log(result);
|
|
156
|
+
// [ { word: '升职', weight: 11.739204307083542 },
|
|
157
|
+
// { word: '加薪', weight: 10.8561552143 },
|
|
158
|
+
// { word: 'CEO', weight: 10.642581114 },
|
|
159
|
+
// { word: '手扶拖拉机', weight: 10.0088573539 },
|
|
160
|
+
// { word: '巅峰', weight: 9.49395840471 } ]
|
|
161
|
+
```
|
|
162
|
+
|
|
163
|
+
### TextRank关键词提取
|
|
164
|
+
|
|
165
|
+
```js
|
|
166
|
+
var nodejieba = require("nodejieba");
|
|
167
|
+
var sentence = "我是拖拉机学院手扶拖拉机专业的。不用多久,我就会升职加薪,当上CEO,走上人生巅峰。";
|
|
168
|
+
var result = nodejieba.textRankExtract(sentence, 5);
|
|
169
|
+
console.log(result);
|
|
170
|
+
// [ { word: '当上', weight: 1 },
|
|
171
|
+
// { word: '不用', weight: 0.9897190043 },
|
|
172
|
+
// { word: '多久', weight: 0.9897190043 },
|
|
173
|
+
// { word: '加薪', weight: 0.9897190043 },
|
|
174
|
+
// { word: '升职', weight: 0.9897190043 } ]
|
|
175
|
+
```
|
|
176
|
+
|
|
177
|
+
### 添加自定义词语
|
|
178
|
+
|
|
179
|
+
```js
|
|
180
|
+
var nodejieba = require("nodejieba");
|
|
181
|
+
console.log(nodejieba.cut("男默女泪"));
|
|
182
|
+
// ["男默", "女泪"]
|
|
183
|
+
nodejieba.insertWord("男默女泪");
|
|
184
|
+
console.log(nodejieba.cut("男默女泪"));
|
|
185
|
+
// ["男默女泪"]
|
|
186
|
+
```
|
|
187
|
+
|
|
188
|
+
More Detals in [demo](https://github.com/yanyiwu/nodejieba-demo)
|
|
189
|
+
|
|
190
|
+
### 关键词抽取
|
|
191
|
+
|
|
192
|
+
```js
|
|
193
|
+
var nodejieba = require("nodejieba");
|
|
194
|
+
var topN = 4;
|
|
195
|
+
console.log(nodejieba.extract("升职加薪,当上CEO,走上人生巅峰。", topN));
|
|
196
|
+
//[ { word: 'CEO', weight: 11.739204307083542 },
|
|
197
|
+
// { word: '升职', weight: 10.8561552143 },
|
|
198
|
+
// { word: '加薪', weight: 10.642581114 },
|
|
199
|
+
// { word: '巅峰', weight: 9.49395840471 } ]
|
|
200
|
+
|
|
201
|
+
console.log(nodejieba.textRankExtract("升职加薪,当上CEO,走上人生巅峰。", topN));
|
|
202
|
+
//[ { word: '当上', weight: 1 },
|
|
203
|
+
// { word: '不用', weight: 0.9898479330698993 },
|
|
204
|
+
// { word: '多久', weight: 0.9851260595435759 },
|
|
205
|
+
// { word: '加薪', weight: 0.9830464899847804 },
|
|
206
|
+
// { word: '升职', weight: 0.9802777682279076 } ]
|
|
207
|
+
```
|
|
208
|
+
|
|
209
|
+
More Detals in [demo](https://github.com/yanyiwu/nodejieba-demo)
|
|
210
|
+
|
|
211
|
+
## Develop NodeJieba
|
|
212
|
+
|
|
213
|
+
```sh
|
|
214
|
+
git clone --recurse-submodules https://github.com/yanyiwu/nodejieba.git
|
|
215
|
+
cd nodejieba
|
|
216
|
+
npm install
|
|
217
|
+
npm test
|
|
218
|
+
```
|
|
219
|
+
|
|
220
|
+
## 应用
|
|
221
|
+
|
|
222
|
+
+ 支持中文搜索的 gitbook 插件: [gitbook-plugin-search-pro]
|
|
223
|
+
+ 汉字拼音转换工具: [pinyin]
|
|
224
|
+
|
|
225
|
+
## 性能评测
|
|
226
|
+
|
|
227
|
+
应该是目前性能最好的 Node.js 中文分词库
|
|
228
|
+
详见: [Jieba中文分词系列性能评测]
|
|
229
|
+
|
|
230
|
+
|
|
231
|
+
[由NodeJieba谈谈Node.js异步实现]:https://github.com/yanyiwu/blog/blob/posts2023archive/_posts/2015-03-21-nodejs-asynchronous-insight.md
|
|
232
|
+
[Node.js的C++扩展初体验之NodeJieba]:https://github.com/yanyiwu/blog/blob/posts2023archive/_posts/2014-02-22-nodejs-cpp-addon-nodejieba.md
|
|
233
|
+
[CppJieba]:https://github.com/yanyiwu/cppjieba.git
|
|
234
|
+
[cnpm]:http://cnpmjs.org
|
|
235
|
+
[Jieba中文分词]:https://github.com/fxsjy/jieba
|
|
236
|
+
|
|
237
|
+
[Jieba中文分词系列性能评测]:https://github.com/yanyiwu/blog/blob/posts2023archive/_posts/2015-06-14-jieba-series-performance-test.md
|
|
238
|
+
[contributors]:https://github.com/yanyiwu/nodejieba/graphs/contributors
|
|
239
|
+
[YanyiWu]:http://github.com/yanyiwu
|
|
240
|
+
[gitbook-plugin-search-pro]:https://plugins.gitbook.com/plugin/search-pro
|
|
241
|
+
[pinyin]:https://github.com/hotoo/pinyin
|
|
242
|
+
|
|
243
|
+
## Contributors
|
|
244
|
+
|
|
245
|
+
### Code Contributors
|
|
246
|
+
|
|
247
|
+
This project exists thanks to all the people who contribute. [[Contribute](CONTRIBUTING.md)].
|
|
248
|
+
<a href="https://github.com/yanyiwu/nodejieba/graphs/contributors"><img src="https://opencollective.com/nodejieba/contributors.svg?width=890&button=false" /></a>
|
|
249
|
+
|
|
250
|
+
### Financial Contributors
|
|
251
|
+
|
|
252
|
+
Become a financial contributor and help us sustain our community. [[Contribute](https://opencollective.com/nodejieba/contribute)]
|
|
253
|
+
|
|
254
|
+
#### Individuals
|
|
255
|
+
|
|
256
|
+
<a href="https://opencollective.com/nodejieba"><img src="https://opencollective.com/nodejieba/individuals.svg?width=890"></a>
|
|
257
|
+
|
|
258
|
+
#### Organizations
|
|
259
|
+
|
|
260
|
+
Support this project with your organization. Your logo will show up here with a link to your website. [[Contribute](https://opencollective.com/nodejieba/contribute)]
|
|
261
|
+
|
|
262
|
+
<a href="https://opencollective.com/nodejieba/organization/0/website"><img src="https://opencollective.com/nodejieba/organization/0/avatar.svg"></a>
|
|
263
|
+
<a href="https://opencollective.com/nodejieba/organization/1/website"><img src="https://opencollective.com/nodejieba/organization/1/avatar.svg"></a>
|
|
264
|
+
<a href="https://opencollective.com/nodejieba/organization/2/website"><img src="https://opencollective.com/nodejieba/organization/2/avatar.svg"></a>
|
|
265
|
+
<a href="https://opencollective.com/nodejieba/organization/3/website"><img src="https://opencollective.com/nodejieba/organization/3/avatar.svg"></a>
|
|
266
|
+
<a href="https://opencollective.com/nodejieba/organization/4/website"><img src="https://opencollective.com/nodejieba/organization/4/avatar.svg"></a>
|
|
267
|
+
<a href="https://opencollective.com/nodejieba/organization/5/website"><img src="https://opencollective.com/nodejieba/organization/5/avatar.svg"></a>
|
|
268
|
+
<a href="https://opencollective.com/nodejieba/organization/6/website"><img src="https://opencollective.com/nodejieba/organization/6/avatar.svg"></a>
|
|
269
|
+
<a href="https://opencollective.com/nodejieba/organization/7/website"><img src="https://opencollective.com/nodejieba/organization/7/avatar.svg"></a>
|
|
270
|
+
<a href="https://opencollective.com/nodejieba/organization/8/website"><img src="https://opencollective.com/nodejieba/organization/8/avatar.svg"></a>
|
|
271
|
+
<a href="https://opencollective.com/nodejieba/organization/9/website"><img src="https://opencollective.com/nodejieba/organization/9/avatar.svg"></a>
|