nodejieba-plus 3.5.10 → 3.5.11

package/README.md

@@ -27,6 +27,7 @@
 + **支持包含空格的关键词**（如 "Open Claw"）。
 + **支持无空格版本匹配**（如 "OpenClaw" 可匹配 "Open Claw"）。
 + **支持英文大小写不敏感匹配**（如 "open claw"、"OPEN CLAW" 都可匹配 "Open Claw"）。
++ **支持批量加载用户词典**（字符串数组、单个字符串、Buffer 格式）。
 
 对实现细节感兴趣的请看如下博文：
 
@@ -188,6 +189,47 @@ console.log(nodejieba.cut("男默女泪"));
 // ["男默女泪"]
 ```
 
+### 批量加载用户词典（新功能）
+
+支持通过字符串数组、单个字符串或 Buffer 批量加载用户词典：
+
+```js
+var nodejieba = require("nodejieba");
+nodejieba.load();
+
+// 方式1：使用字符串数组
+nodejieba.loadUserDict(["云计算", "人工智能 1000 nz", "大数据"]);
+
+// 方式2：使用单个字符串
+nodejieba.loadUserDict("区块链");
+
+// 方式3：使用 Buffer
+const dictBuffer = Buffer.from("新词1\n新词2 100 n\n新词3 nz");
+nodejieba.loadUserDict(dictBuffer);
+
+// 分词时会识别用户词典中的词
+var result = nodejieba.cut("云计算和大数据是人工智能的基础");
+console.log(result); // ['云计算', '和', '大数据', '是', '人工智能', '的', '基础']
+```
+
+#### 词典条目格式
+
+词典条目支持以下格式：
+
+```
+# 仅关键词
+云计算
+
+# 关键词 + 词频
+人工智能 1000
+
+# 关键词 + 词性标签
+区块链 nz
+
+# 关键词 + 词频 + 词性标签
+大数据 500 n
+```
+
 ### 包含空格的关键词（新功能）
 
 支持在自定义词典中使用包含空格的关键词，且支持无空格版本匹配和大小写不敏感匹配。

package/index.js

@@ -73,5 +73,6 @@ wrapWithDictLoad("tag");
 wrapWithDictLoad("extract");
 wrapWithDictLoad("textRankExtract");
 wrapWithDictLoad("insertWord");
+wrapWithDictLoad("loadUserDict");
 
 module.exports = exports;

package/lib/nodejieba.cpp

@@ -6,6 +6,8 @@
 #include "cppjieba/KeywordExtractor.hpp"
 #include "cppjieba/TextRankExtractor.hpp"
 
+#include <sstream>
+
 NodeJieba::NodeJieba(Napi::Env env, Napi::Object exports) {
   DefineAddon(exports, {
     InstanceMethod("load", &NodeJieba::load),
@@ -17,7 +19,8 @@ NodeJieba::NodeJieba(Napi::Env env, Napi::Object exports) {
     InstanceMethod("tag", &NodeJieba::tag),
     InstanceMethod("extract", &NodeJieba::extract),
     InstanceMethod("textRankExtract", &NodeJieba::textRankExtract),
-    InstanceMethod("insertWord", &NodeJieba::insertWord)
+    InstanceMethod("insertWord", &NodeJieba::insertWord),
+    InstanceMethod("loadUserDict", &NodeJieba::loadUserDict)
   });
 }
 
@@ -216,3 +219,49 @@ Napi::Value NodeJieba::textRankExtract(const Napi::CallbackInfo& info) {
   WrapPairVector(info.Env(), words, outArray);
   return outArray;
 }
+
+Napi::Value NodeJieba::loadUserDict(const Napi::CallbackInfo& info) {
+  if (info.Length() < 1) {
+    return Napi::Boolean::New(info.Env(), false);
+  }
+
+  if( !_jieba_handle ){
+    Napi::Error::New(info.Env(), "Before calling any other function you have to call load() first").ThrowAsJavaScriptException();
+  }
+
+  // 支持传入字符串数组或单个字符串（Buffer）
+  if (info[0].IsArray()) {
+    Napi::Array arr = info[0].As<Napi::Array>();
+    std::vector<std::string> buf;
+    for (size_t i = 0; i < arr.Length(); i++) {
+      Napi::Value val = arr[i];
+      if (val.IsString()) {
+        buf.push_back(val.As<Napi::String>().Utf8Value());
+      }
+    }
+    _jieba_handle->LoadUserDict(buf);
+  } else if (info[0].IsString()) {
+    // 支持传入单个词典条目字符串
+    std::string line = info[0].As<Napi::String>().Utf8Value();
+    std::vector<std::string> buf;
+    buf.push_back(line);
+    _jieba_handle->LoadUserDict(buf);
+  } else if (info[0].IsBuffer()) {
+    // 支持传入 Buffer，将其转换为字符串并按行分割
+    Napi::Buffer<char> buffer = info[0].As<Napi::Buffer<char>>();
+    std::string content(buffer.Data(), buffer.Length());
+    std::vector<std::string> buf;
+    std::istringstream iss(content);
+    std::string line;
+    while (std::getline(iss, line)) {
+      if (!line.empty()) {
+        buf.push_back(line);
+      }
+    }
+    _jieba_handle->LoadUserDict(buf);
+  } else {
+    return Napi::Boolean::New(info.Env(), false);
+  }
+
+  return Napi::Boolean::New(info.Env(), true);
+}

package/lib/nodejieba.h

@@ -20,6 +20,7 @@ private:
   Napi::Value extract(const Napi::CallbackInfo& info);
   Napi::Value textRankExtract(const Napi::CallbackInfo& info);
   Napi::Value insertWord(const Napi::CallbackInfo& info);
+  Napi::Value loadUserDict(const Napi::CallbackInfo& info);
 
   cppjieba::Jieba* _jieba_handle{nullptr};
   cppjieba::TextRankExtractor* _text_rank_extractor_handle{nullptr};

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "nodejieba-plus",
   "description": "chinese word segmentation for node",
-  "version": "3.5.10",
+  "version": "3.5.11",
   "author": "Yanyi Wu <wuyanyi09@foxmail.com>",
   "maintainers": [
     "Yanyi Wu <wuyanyi09@foxmail.com>"
@@ -33,7 +33,7 @@
     "typescript": "^5.0.4"
   },
   "scripts": {
-    "test": "mocha --timeout 10s -R spec test/test.js && mocha --timeout 10s -R spec test/load_dict_test.js && mocha --timeout 10s -R spec test/missing_binding_test.js",
+    "test": "mocha --timeout 10s -R spec test/test.js && mocha --timeout 10s -R spec test/load_dict_test.js && mocha --timeout 10s -R spec test/load_user_dict_test.js && mocha --timeout 10s -R spec test/missing_binding_test.js",
     "install": "npx @mapbox/node-pre-gyp install --fallback-to-build",
     "rebuild": "npx @mapbox/node-pre-gyp rebuild"
   },

package/test/load_user_dict_test.js

@@ -0,0 +1,58 @@
+var should = require("should");
+var nodejieba = require("../index.js");
+
+describe("nodejieba.loadUserDict", function() {
+
+  // 确保在测试前加载词典
+  before(function() {
+    nodejieba.load();
+  });
+
+  it("nodejieba.loadUserDict with string array should return true", function() {
+    // 使用字符串数组加载用户词典
+    var dictLines = [
+      "测试新词",
+      "自定义词汇 10 n"
+    ];
+    var loadResult = nodejieba.loadUserDict(dictLines);
+    loadResult.should.eql(true);
+  });
+
+  it("nodejieba.loadUserDict with single string should return true", function() {
+    // 加载单个词典条目
+    var loadResult = nodejieba.loadUserDict("单个词");
+    loadResult.should.eql(true);
+  });
+
+  it("nodejieba.loadUserDict with Buffer should return true", function() {
+    // 使用 Buffer 加载用户词典
+    var dictContent = Buffer.from("Buffer词 100 nz\n另一个词");
+    var loadResult = nodejieba.loadUserDict(dictContent);
+    loadResult.should.eql(true);
+  });
+
+  it("nodejieba.loadUserDict should return false when no argument", function() {
+    var loadResult = nodejieba.loadUserDict();
+    loadResult.should.eql(false);
+  });
+
+  it("nodejieba.loadUserDict with tag should return true", function() {
+    // 测试带标签的词典条目
+    var dictLines = [
+      "技术术语 500 nz"
+    ];
+    var result = nodejieba.loadUserDict(dictLines);
+    result.should.eql(true);
+  });
+
+  it("nodejieba.loadUserDict should work after loading words", function() {
+    // 先加载一些用户词典 - 使用比较独特的词确保测试准确性
+    nodejieba.loadUserDict(["云计算"]);
+    nodejieba.loadUserDict("人工智能 1000 nz");
+    
+    // 验证这些词被正确识别
+    var result = nodejieba.cut("云计算是人工智能的基础");
+    result.should.containEql('云计算');
+    result.should.containEql('人工智能');
+  });
+});

package/test_open_claw.js

@@ -0,0 +1,65 @@
+// 测试 "open claw" 关键词提取问题
+
+var nodejieba = require("./index.js");
+
+// 测试句子
+var sentence = "Node.js在Web开发中的应用与实践Open Claw，这句测试的话，关键词是\"open claw\"";
+
+console.log("=".repeat(60));
+console.log("测试句子:", sentence);
+console.log("=".repeat(60));
+
+// 1. 先进行分词测试
+console.log("\n【1. 分词结果】");
+var cutResult = nodejieba.cut(sentence);
+console.log("cut:", cutResult);
+
+// 2. 关键词提取测试
+console.log("\n【2. 关键词提取 (extract)】");
+var keywords = nodejieba.extract(sentence, 10);
+console.log("提取的关键词:");
+keywords.forEach(function(kw) {
+  console.log("  - " + kw.word + " (权重: " + kw.weight + ")");
+});
+
+// 3. TextRank 关键词提取测试
+console.log("\n【3. TextRank 关键词提取】");
+var textRankKeywords = nodejieba.textRankExtract(sentence, 10);
+console.log("提取的关键词:");
+textRankKeywords.forEach(function(kw) {
+  console.log("  - " + kw.word + " (权重: " + kw.weight + ")");
+});
+
+// 4. 检查是否包含 "open claw"
+console.log("\n【4. 检查结果】");
+var hasOpenClaw = keywords.some(function(kw) {
+  return kw.word.toLowerCase() === "open claw";
+});
+console.log("是否提取到 'open claw':", hasOpenClaw);
+
+// 5. 添加自定义词后再次测试
+console.log("\n【5. 添加自定义词后测试】");
+nodejieba.insertWord("open claw");
+console.log("已添加自定义词: open claw");
+
+var cutResult2 = nodejieba.cut(sentence);
+console.log("\n再次分词结果:");
+console.log("cut:", cutResult2);
+
+var keywords2 = nodejieba.extract(sentence, 10);
+console.log("\n再次提取关键词:");
+keywords2.forEach(function(kw) {
+  console.log("  - " + kw.word + " (权重: " + kw.weight + ")");
+});
+
+var hasOpenClaw2 = keywords2.some(function(kw) {
+  return kw.word.toLowerCase() === "open claw";
+});
+console.log("\n是否提取到 'open claw':", hasOpenClaw2);
+
+console.log("\n" + "=".repeat(60));
+console.log("问题分析:");
+console.log("1. jieba 分词器默认基于中文语料训练，对英文词汇识别有限");
+console.log("2. 'Open Claw' 作为英文词组，默认词典中不存在");
+console.log("3. 解决方案: 使用 insertWord() 方法添加自定义词");
+console.log("=".repeat(60));

package/types/index.d.ts

@@ -27,4 +27,5 @@ declare module "nodejieba" {
   export function textRankExtract(sentence: string, threshold: number): ExtractResult[];
   export function insertWord(word: string, tag?: string): boolean;
   export function cutSmall(sentence: string, small: number): string[];
+  export function loadUserDict(dict: string | string[] | Buffer): boolean;
 }