nodejieba-plus 3.5.8

package/README.md

@@ -0,0 +1,349 @@
+[![Build Status](https://github.com/yanyiwu/nodejieba/actions/workflows/test.yml/badge.svg)](https://github.com/yanyiwu/nodejieba/actions/workflows/test.yml)
+[![Financial Contributors on Open Collective](https://opencollective.com/nodejieba/all/badge.svg?label=financial+contributors)](https://opencollective.com/nodejieba) [![Author](https://img.shields.io/badge/author-@yanyiwu-blue.svg?style=flat)](https://github.com/yanyiwu/) 
+[![Platform](https://img.shields.io/badge/platform-Linux,macOS,Windows-green.svg?style=flat)](https://github.com/yanyiwu/nodejieba)
+[![Performance](https://img.shields.io/badge/performance-excellent-brightgreen.svg?style=flat)](https://github.com/yanyiwu/blog/blob/posts2023archive/_posts/2015-06-14-jieba-series-performance-test.md) 
+[![License](https://img.shields.io/badge/license-MIT-yellow.svg?style=flat)](http://yanyiwu.mit-license.org)
+[![NpmDownload Status](http://img.shields.io/npm/dm/nodejieba.svg)](https://www.npmjs.org/package/nodejieba)
+[![NPM Version](https://img.shields.io/npm/v/nodejieba.svg?style=flat)](https://www.npmjs.org/package/nodejieba)
+[![Code Climate](https://codeclimate.com/github/yanyiwu/nodejieba/badges/gpa.svg)](https://codeclimate.com/github/yanyiwu/nodejieba)
+
+- - -
+
+# NodeJieba "结巴"分词的Node.js版本
+
+## 介绍 
+
+`NodeJieba`是"结巴"中文分词的 Node.js 版本实现，
+由[CppJieba]提供底层分词算法实现，
+是兼具高性能和易用性两者的 Node.js 中文分词组件。
+
+## 特点
+
++ 词典载入方式灵活，无需配置词典路径也可使用，需要定制自己的词典路径时也可灵活定制。
++ 底层算法实现是C++，性能高效。
++ 支持多种分词算法，各种分词算法见[CppJieba]的README.md介绍。
++ 支持动态补充词库。
++ 支持TypeScript，提供完整的类型定义。
++ **支持包含空格的关键词**（如 "Open Claw"）。
++ **支持无空格版本匹配**（如 "OpenClaw" 可匹配 "Open Claw"）。
++ **支持英文大小写不敏感匹配**（如 "open claw"、"OPEN CLAW" 都可匹配 "Open Claw"）。
+
+对实现细节感兴趣的请看如下博文：
+
++ [Node.js的C++扩展初体验之NodeJieba] 
++ [由NodeJieba谈谈Node.js异步实现] 
+
+## 安装
+
+```sh
+npm install nodejieba
+```
+
+## 快速开始
+
+```js
+var nodejieba = require("nodejieba");
+var result = nodejieba.cut("南京市长江大桥");
+console.log(result);
+//["南京市","长江大桥"]
+```
+
+更多示例请参考 [demo](https://github.com/yanyiwu/nodejieba-demo)
+
+### 词典载入可灵活配置
+
+如果没有主动调用词典函数时，
+则会在第一次调用cut等功能函数时，自动载入默认词典。
+
+如果要主动触发词典载入，则使用以下函数主动触发。
+
+```js
+nodejieba.load();
+```
+
+以上用法会自动载入所有默认词典，
+如果需要载入自己的词典，而不是默认词典。
+比如想要载入自己的用户词典，则使用以下函数：
+
+```js
+nodejieba.load({
+  userDict: './test/testdata/userdict.utf8',
+});
+```
+
+字典载入函数load的参数项都是可选的，
+如果没有对应的项则自动填充默认参数。
+所以上面这段代码和下面这代代码是等价的。
+
+```js
+nodejieba.load({
+  dict: nodejieba.DEFAULT_DICT,
+  hmmDict: nodejieba.DEFAULT_HMM_DICT,
+  userDict: './test/testdata/userdict.utf8',
+  idfDict: nodejieba.DEFAULT_IDF_DICT,
+  stopWordDict: nodejieba.DEFAULT_STOP_WORD_DICT,
+});
+```
+
+#### 词典说明
+
++ dict: 主词典，带权重和词性标签，建议使用默认词典。
++ hmmDict: 隐式马尔科夫模型，建议使用默认词典。
++ userDict: 用户词典，建议自己根据需要定制。
++ idfDict: 关键词抽取所需的idf信息。
++ stopWordDict: 关键词抽取所需的停用词列表。
+
+## API 文档
+
+### 分词
+
+#### 1. 默认分词
+
+```js
+var nodejieba = require("nodejieba");
+var result = nodejieba.cut("南京市长江大桥");
+console.log(result);
+// ["南京市", "长江大桥"]
+```
+
+#### 2. 使用HMM模型分词
+
+```js
+var result = nodejieba.cutHMM("南京市长江大桥");
+console.log(result);
+// ["南京市", "长江大桥"]
+```
+
+#### 3. 全模式分词
+
+```js
+var result = nodejieba.cutAll("南京市长江大桥");
+console.log(result);
+// ["南京", "南京市", "市长", "长江", "长江大桥", "大桥"]
+```
+
+#### 4. 搜索引擎模式分词
+
+```js
+var result = nodejieba.cutForSearch("南京市长江大桥");
+console.log(result);
+// ["南京", "市", "长江", "大桥", "南京市", "长江大桥"]
+```
+
+#### 5. 小粒度分词
+
+```js
+var result = nodejieba.cutSmall("南京市长江大桥", 3);
+console.log(result);
+// ["南京市", "长江大桥"]
+```
+
+### 词性标注
+
+```js
+var nodejieba = require("nodejieba");
+var result = nodejieba.tag("红掌拨清波");
+console.log(result);
+// [ { word: '红掌', tag: 'n' },
+//   { word: '拨', tag: 'v' },
+//   { word: '清波', tag: 'n' } ]
+```
+
+### 关键词提取
+
+```js
+var nodejieba = require("nodejieba");
+var sentence = "我是拖拉机学院手扶拖拉机专业的。不用多久，我就会升职加薪，当上CEO，走上人生巅峰。";
+var result = nodejieba.extract(sentence, 5);
+console.log(result);
+// [ { word: '升职', weight: 11.739204307083542 },
+//   { word: '加薪', weight: 10.8561552143 },
+//   { word: 'CEO', weight: 10.642581114 },
+//   { word: '手扶拖拉机', weight: 10.0088573539 },
+//   { word: '巅峰', weight: 9.49395840471 } ]
+```
+
+### TextRank关键词提取
+
+```js
+var nodejieba = require("nodejieba");
+var sentence = "我是拖拉机学院手扶拖拉机专业的。不用多久，我就会升职加薪，当上CEO，走上人生巅峰。";
+var result = nodejieba.textRankExtract(sentence, 5);
+console.log(result);
+// [ { word: '当上', weight: 1 },
+//   { word: '不用', weight: 0.9897190043 },
+//   { word: '多久', weight: 0.9897190043 },
+//   { word: '加薪', weight: 0.9897190043 },
+//   { word: '升职', weight: 0.9897190043 } ]
+```
+
+### 添加自定义词语
+
+```js
+var nodejieba = require("nodejieba");
+console.log(nodejieba.cut("男默女泪"));
+// ["男默", "女泪"]
+nodejieba.insertWord("男默女泪");
+console.log(nodejieba.cut("男默女泪"));
+// ["男默女泪"]
+```
+
+### 包含空格的关键词（新功能）
+
+支持在自定义词典中使用包含空格的关键词，且支持无空格版本匹配和大小写不敏感匹配。
+
+#### 用户词典格式
+
+用户词典支持以下格式：
+
+```
+# 只有关键词
+Open Claw
+
+# 关键词 + 词性标签
+Open Claw n
+
+# 关键词 + 词频 + 词性标签
+Open Claw 100 n
+
+# 包含多个空格的关键词
+Machine Learning 200 n
+Artificial Intelligence 300 n
+```
+
+#### 使用示例
+
+```js
+var nodejieba = require("nodejieba");
+var fs = require('fs');
+var path = require('path');
+
+// 创建包含空格关键词的用户词典
+var dictContent = `Open Claw 100 n
+Machine Learning 200 n
+Artificial Intelligence 300 n
+`;
+
+var testDictPath = path.join(__dirname, 'user_dict.utf8');
+fs.writeFileSync(testDictPath, dictContent);
+
+// 加载词典
+nodejieba.load({
+  userDict: testDictPath,
+});
+
+// 测试1: 包含空格的关键词匹配
+console.log(nodejieba.cut("I want to use Open Claw tool"));
+// 输出包含: ['Open Claw']
+
+// 测试2: 大小写不敏感匹配
+console.log(nodejieba.cut("open claw"));        // 匹配 Open Claw
+console.log(nodejieba.cut("OPEN CLAW"));        // 匹配 Open Claw
+console.log(nodejieba.cut("Open Claw"));        // 匹配 Open Claw
+
+// 测试3: 无空格版本匹配
+console.log(nodejieba.cut("OpenClaw"));         // 匹配 Open Claw
+console.log(nodejieba.cut("openclaw"));         // 匹配 Open Claw
+console.log(nodejieba.cut("OPENCLAW"));         // 匹配 Open Claw
+
+// 测试4: 其他包含空格的关键词
+console.log(nodejieba.cut("Machine Learning is great"));
+// 输出包含: ['Machine Learning']
+
+console.log(nodejieba.cut("Artificial Intelligence will change the world"));
+// 输出包含: ['Artificial Intelligence']
+
+// 清理测试文件
+fs.unlinkSync(testDictPath);
+```
+
+#### 功能说明
+
+1. **包含空格的关键词**: 词典中的 "Open Claw" 可以匹配文本中的 "Open Claw"
+2. **无空格版本匹配**: 词典中的 "Open Claw" 也可以匹配文本中的 "OpenClaw"
+3. **大小写不敏感**: 词典中的 "Open Claw" 可以匹配 "open claw"、"OPEN CLAW"、"Open Claw" 等任意大小写组合
+
+More Detals in [demo](https://github.com/yanyiwu/nodejieba-demo)
+
+### 关键词抽取
+
+```js
+var nodejieba = require("nodejieba");
+var topN = 4;
+console.log(nodejieba.extract("升职加薪，当上CEO，走上人生巅峰。", topN));
+//[ { word: 'CEO', weight: 11.739204307083542 },
+//  { word: '升职', weight: 10.8561552143 },
+//  { word: '加薪', weight: 10.642581114 },
+//  { word: '巅峰', weight: 9.49395840471 } ]
+
+console.log(nodejieba.textRankExtract("升职加薪，当上CEO，走上人生巅峰。", topN));
+//[ { word: '当上', weight: 1 },
+//  { word: '不用', weight: 0.9898479330698993 },
+//  { word: '多久', weight: 0.9851260595435759 },
+//  { word: '加薪', weight: 0.9830464899847804 },
+//  { word: '升职', weight: 0.9802777682279076 } ]
+```
+
+More Detals in [demo](https://github.com/yanyiwu/nodejieba-demo)
+
+## Develop NodeJieba
+
+```sh
+git clone --recurse-submodules https://github.com/yanyiwu/nodejieba.git
+cd nodejieba
+npm install
+npm test
+```
+
+## 应用
+
++ 支持中文搜索的 gitbook 插件: [gitbook-plugin-search-pro]
++ 汉字拼音转换工具: [pinyin]
+
+## 性能评测
+
+应该是目前性能最好的 Node.js 中文分词库
+详见: [Jieba中文分词系列性能评测]
+
+
+[由NodeJieba谈谈Node.js异步实现]:https://github.com/yanyiwu/blog/blob/posts2023archive/_posts/2015-03-21-nodejs-asynchronous-insight.md
+[Node.js的C++扩展初体验之NodeJieba]:https://github.com/yanyiwu/blog/blob/posts2023archive/_posts/2014-02-22-nodejs-cpp-addon-nodejieba.md
+[CppJieba]:https://github.com/yanyiwu/cppjieba.git
+[cnpm]:http://cnpmjs.org
+[Jieba中文分词]:https://github.com/fxsjy/jieba
+
+[Jieba中文分词系列性能评测]:https://github.com/yanyiwu/blog/blob/posts2023archive/_posts/2015-06-14-jieba-series-performance-test.md
+[contributors]:https://github.com/yanyiwu/nodejieba/graphs/contributors
+[YanyiWu]:http://github.com/yanyiwu
+[gitbook-plugin-search-pro]:https://plugins.gitbook.com/plugin/search-pro
+[pinyin]:https://github.com/hotoo/pinyin
+
+## Contributors
+
+### Code Contributors
+
+This project exists thanks to all the people who contribute. [[Contribute](CONTRIBUTING.md)].
+<a href="https://github.com/yanyiwu/nodejieba/graphs/contributors"><img src="https://opencollective.com/nodejieba/contributors.svg?width=890&button=false" /></a>
+
+### Financial Contributors
+
+Become a financial contributor and help us sustain our community. [[Contribute](https://opencollective.com/nodejieba/contribute)]
+
+#### Individuals
+
+<a href="https://opencollective.com/nodejieba"><img src="https://opencollective.com/nodejieba/individuals.svg?width=890"></a>
+
+#### Organizations
+
+Support this project with your organization. Your logo will show up here with a link to your website. [[Contribute](https://opencollective.com/nodejieba/contribute)]
+
+<a href="https://opencollective.com/nodejieba/organization/0/website"><img src="https://opencollective.com/nodejieba/organization/0/avatar.svg"></a>
+<a href="https://opencollective.com/nodejieba/organization/1/website"><img src="https://opencollective.com/nodejieba/organization/1/avatar.svg"></a>
+<a href="https://opencollective.com/nodejieba/organization/2/website"><img src="https://opencollective.com/nodejieba/organization/2/avatar.svg"></a>
+<a href="https://opencollective.com/nodejieba/organization/3/website"><img src="https://opencollective.com/nodejieba/organization/3/avatar.svg"></a>
+<a href="https://opencollective.com/nodejieba/organization/4/website"><img src="https://opencollective.com/nodejieba/organization/4/avatar.svg"></a>
+<a href="https://opencollective.com/nodejieba/organization/5/website"><img src="https://opencollective.com/nodejieba/organization/5/avatar.svg"></a>
+<a href="https://opencollective.com/nodejieba/organization/6/website"><img src="https://opencollective.com/nodejieba/organization/6/avatar.svg"></a>
+<a href="https://opencollective.com/nodejieba/organization/7/website"><img src="https://opencollective.com/nodejieba/organization/7/avatar.svg"></a>
+<a href="https://opencollective.com/nodejieba/organization/8/website"><img src="https://opencollective.com/nodejieba/organization/8/avatar.svg"></a>
+<a href="https://opencollective.com/nodejieba/organization/9/website"><img src="https://opencollective.com/nodejieba/organization/9/avatar.svg"></a>

package/binding.gyp

@@ -0,0 +1,63 @@
+{
+  "targets": [
+    {
+      "target_name": "nodejieba",
+      "cflags!": [ "-fno-exceptions" ],
+      "cflags_cc!": [ "-fno-exceptions" ],
+      "xcode_settings": {
+        "GCC_ENABLE_CPP_EXCEPTIONS": "YES",
+        "CLANG_CXX_LIBRARY": "libc++",
+        "MACOSX_DEPLOYMENT_TARGET": "10.13",
+      },
+      "msvs_settings": {
+        "VCCLCompilerTool": {
+          "ExceptionHandling": 1,
+          "AdditionalOptions": ["/std:c++14","/utf-8"]
+        },
+      },
+      "win_delay_load_hook": "true",
+      "sources": [
+        "./lib/index.cpp",
+        "./lib/nodejieba.cpp",
+      ],
+      "cflags": [
+        "-DLOGGING_LEVEL=LL_WARNING"
+      ],
+      "include_dirs" : [
+        "<!(node -p \"require('node-addon-api').include_dir\")",
+        "./submodules/cppjieba/include",
+        "./submodules/cppjieba/deps/limonp/include",
+      ],
+      'configurations': {
+        'Release': {
+          'msvs_settings': {
+            'VCCLCompilerTool': {
+              'ExceptionHandling': '1',
+              'PreprocessorDefinitions': ['LOGGING_LEVEL=LL_WARNING'],
+            }
+          }
+        }
+      },
+      "conditions": [
+        [ "OS == 'mac'", {
+          "xcode_settings": {
+            "OTHER_CPLUSPLUSFLAGS":[
+              "-mmacosx-version-min=10.13",
+              "-std=c++14",
+              "-stdlib=libc++",
+              "-DLOGGING_LEVEL=LL_WARNING",
+            ]
+          }
+        }],
+        # 添加Windows特定条件
+        [ "OS == 'win'", {
+          "msvs_settings": {
+            "VCCLCompilerTool": {
+              "AdditionalOptions": ["/std:c++14"]
+            }
+          }
+        }]
+      ],
+    }
+  ]
+}

package/index.js

@@ -0,0 +1,77 @@
+var path = require('path');
+var BINARY_PATH = path.join(__dirname, 'build/Release/nodejieba.node');
+var nodejieba;
+var bindingLoadError;
+
+function getNodejieba() {
+  if (nodejieba) {
+    return nodejieba;
+  }
+  if (bindingLoadError) {
+    throw bindingLoadError;
+  }
+  try {
+    nodejieba = require(BINARY_PATH);
+    return nodejieba;
+  } catch (err) {
+    if (err && err.code === 'MODULE_NOT_FOUND') {
+      bindingLoadError = new Error(
+        "nodejieba native binding was not found at " + BINARY_PATH + ". " +
+        "This usually means install scripts were skipped or the native binary failed to download/build. " +
+        "Try reinstalling without --ignore-scripts or run `npm rebuild nodejieba`."
+      );
+      bindingLoadError.code = 'BINDING_NOT_FOUND';
+      bindingLoadError.cause = err;
+      throw bindingLoadError;
+    }
+    throw err;
+  }
+}
+
+var isDictLoaded = false;
+
+var DICT_DIR = __dirname + "/submodules/cppjieba/dict/";
+
+var exports = {
+  DEFAULT_DICT: DICT_DIR + "jieba.dict.utf8",
+  DEFAULT_HMM_DICT: DICT_DIR + "hmm_model.utf8",
+  DEFAULT_USER_DICT: DICT_DIR + "user.dict.utf8",
+  DEFAULT_IDF_DICT: DICT_DIR + "idf.utf8",
+  DEFAULT_STOP_WORD_DICT: DICT_DIR + "stop_words.utf8",
+
+  load: function (dictJson) {
+    if (!dictJson) {
+      dictJson = {};
+    }
+    dict         = dictJson.dict       || exports.DEFAULT_DICT;
+    hmmDict      = dictJson.hmmDict    || exports.DEFAULT_HMM_DICT;
+    userDict     = dictJson.userDict   || exports.DEFAULT_USER_DICT;
+    idfDict      = dictJson.idfDict    || exports.DEFAULT_IDF_DICT;
+    stopWordDict = dictJson.stopWordDict || exports.DEFAULT_STOP_WORD_DICT;
+
+    var result = getNodejieba().load(dict, hmmDict, userDict, idfDict, stopWordDict);
+    isDictLoaded = true;
+    return result;
+  }
+};
+
+function wrapWithDictLoad(functName) {
+  exports[functName] = function () {
+    if (!isDictLoaded) {
+      exports.load();
+    }
+    return getNodejieba()[functName].apply(this, arguments);
+  }
+}
+
+wrapWithDictLoad("cut");
+wrapWithDictLoad("cutAll");
+wrapWithDictLoad("cutHMM");
+wrapWithDictLoad("cutForSearch");
+wrapWithDictLoad("cutSmall");
+wrapWithDictLoad("tag");
+wrapWithDictLoad("extract");
+wrapWithDictLoad("textRankExtract");
+wrapWithDictLoad("insertWord");
+
+module.exports = exports;

package/lib/index.cpp

@@ -0,0 +1,3 @@
+#include "nodejieba.h"
+
+NODE_API_ADDON(NodeJieba)