nodejieba-plus 3.5.16 → 3.5.18

package/README.md

@@ -220,6 +220,68 @@ nodejieba.loadUserDict(dictBuffer);
 // 分词时会识别用户词典中的词
 var result = nodejieba.cut("云计算和大数据是人工智能的基础");
 console.log(result); // ['云计算', '和', '大数据', '是', '人工智能', '的', '基础']
+
+// 关键词提取时，用户词典中的词会自动获得更高的权重（默认2倍）
+var keywords = nodejieba.extract("云计算和大数据是人工智能的基础", 5);
+console.log(keywords); // 用户词典中的词排名会显著提升
+```
+
+#### 用户词典权重提升机制（新功能）
+
+从 v3.5.16 开始，加载用户词典时会自动为词典中的词设置更高的 IDF 权重，确保在关键词提取时获得更高的排名：
+
+**自动权重提升**：
+- 加载用户词典后，词典中的词会自动获得 **2 倍 IDF 权重**
+- 这意味着用户词典中的词在关键词提取时会优先显示
+
+**手动设置权重**：
+```js
+// 方式1：设置具体的 IDF 值
+nodejieba.setIdf("Open Claw", 30.0);
+
+// 方式2：使用倍数提升权重（默认2倍）
+nodejieba.setIdf("Open Claw");  // 2倍权重
+
+// 方式3：自定义倍数
+nodejieba.setIdf("Open Claw", null, 3.0);  // 3倍权重
+```
+
+#### IDF 词典支持空格关键词（新功能）
+
+从 v3.5.16 开始，IDF 词典支持包含空格的关键词：
+
+**IDF 词典格式**：
+```
+# 普通关键词
+互动 12.0
+
+# 包含空格的关键词
+Open Claw 30.0
+Machine Learning 25.0
+Deep Learning 28.0
+```
+
+**使用示例**：
+```js
+var nodejieba = require("nodejieba");
+
+// 加载包含空格关键词的 IDF 词典
+nodejieba.load({
+  idfDict: "./custom_idf.txt"
+});
+
+// 加载用户词典（同时需要分词词典支持）
+nodejieba.loadUserDict(["Open Claw", "Machine Learning", "Deep Learning"]);
+
+// 关键词提取时会正确识别包含空格的词
+var keywords = nodejieba.extract("Open Claw和Machine Learning都是Deep Learning的基础", 5);
+console.log(keywords);
+// 输出: [
+//   { word: 'Open Claw', weight: 30.00 },
+//   { word: 'Deep Learning', weight: 28.00 },
+//   { word: 'Machine Learning', weight: 25.00 },
+//   ...
+// ]
 ```
 
 #### 词典条目格式
@@ -240,6 +302,111 @@ console.log(result); // ['云计算', '和', '大数据', '是', '人工智能',
 大数据 500 n
 ```
 
+### 批量加载IDF词典（新功能）
+
+支持通过字符串数组、Set、单个字符串或 Buffer 批量加载 IDF 词典，用于关键词提取时设置词语的权重。
+
+**功能特点**：
+- 支持多种输入格式：字符串数组、Set、单个字符串、Buffer
+- 支持包含空格的词组（如 "Deep Learning"）
+- 自动适配不含空格的版本（"Deep Learning" 也可匹配 "DeepLearning"）
+- 英文大小写不敏感（"MachineLearning" 可匹配 "machinelearning"、"MACHINELEARNING"）
+- 同时添加到分词词典，确保分词时能正确识别
+
+**使用方法**：
+
+```js
+var nodejieba = require("nodejieba");
+nodejieba.load();
+
+// 方式1：使用字符串数组
+nodejieba.loadIdfDict(['自定义词1 15.0', '自定义词2 12.5', '测试词汇 10.0']);
+
+// 方式2：使用 Set 集合
+const idfSet = new Set(['集合词汇1 14.0', '集合词汇2 13.0']);
+nodejieba.loadIdfDict(idfSet);
+
+// 方式3：使用单个字符串
+nodejieba.loadIdfDict('单个词汇 20.0');
+
+// 方式4：使用 Buffer（必须是 UTF-8 编码）
+const idfBuffer = Buffer.from('缓冲词汇1 18.0\n缓冲词汇2 16.5');
+nodejieba.loadIdfDict(idfBuffer);
+
+// 关键词提取时会使用设置的 IDF 权重
+var keywords = nodejieba.extract('这是一个自定义词1和自定义词2的测试词汇', 3);
+console.log(keywords);
+// 输出: [
+//   { word: '自定义词1', weight: 15 },
+//   { word: '自定义词2', weight: 12.5 },
+//   { word: '测试词汇', weight: 10 }
+// ]
+```
+
+**支持包含空格的词组**：
+
+```js
+// 加载包含空格的词组
+nodejieba.loadIdfDict(['人工智能 技术 25.0', '机器 学习 22.0']);
+
+// 分词时会识别这些词组
+console.log(nodejieba.cut('人工智能技术的发展和机器学习的应用'));
+// 输出: ['人工智能技术', '的', '发展', '和', '机器学习', '的', '应用']
+
+// 关键词提取时使用设置的权重
+var keywords = nodejieba.extract('人工智能技术的发展和机器学习的应用', 5);
+console.log(keywords);
+// 输出: [
+//   { word: '人工智能技术', weight: 25 },
+//   { word: '机器学习', weight: 22 },
+//   ...
+// ]
+```
+
+**英文大小写不敏感**：
+
+```js
+// 加载英文 IDF 词组
+nodejieba.loadIdfDict(['MachineLearning 30.0', 'Deep Learning 28.0', 'AI Technology 26.0']);
+
+// 各种大小写都能正确匹配
+console.log(nodejieba.extract('MachineLearning is important', 2));
+// 输出: [{ word: 'MachineLearning', weight: 30 }, ...]
+
+console.log(nodejieba.extract('machinelearning is important', 2));
+// 输出: [{ word: 'machinelearning', weight: 30 }, ...]
+
+console.log(nodejieba.extract('MACHINELEARNING is important', 2));
+// 输出: [{ word: 'MACHINELEARNING', weight: 30 }, ...]
+
+// 包含空格的英文词组也能正确匹配
+console.log(nodejieba.extract('Deep Learning and AI Technology', 3));
+// 输出: [{ word: 'Deep Learning', weight: 28 }, { word: 'AI Technology', weight: 26 }, ...]
+
+// 不含空格的版本也能匹配
+console.log(nodejieba.extract('DeepLearning and AITechnology', 3));
+// 输出: [{ word: 'DeepLearning', weight: 28 }, { word: 'AITechnology', weight: 26 }, ...]
+```
+
+**IDF 词典条目格式**：
+
+```
+# 词语 IDF权重
+自定义词 15.0
+
+# 包含空格的词组
+Deep Learning 28.0
+Machine Learning 25.0
+
+# 中文词组
+人工智能 技术 25.0
+```
+
+**注意事项**：
+- IDF 值越大，词语在关键词提取时的权重越高
+- 建议根据词语的重要性设置合理的 IDF 值（通常在 5-30 之间）
+- 加载 IDF 词典时会自动将词语添加到分词词典，无需额外调用 `loadUserDict`
+
 ### 包含空格的关键词（新功能）
 
 支持在自定义词典中使用包含空格的关键词，且支持无空格版本匹配和大小写不敏感匹配。

package/analyze_weight.js

@@ -0,0 +1,57 @@
+var nodejieba = require("./index.js");
+
+console.log("=== 关键词提取权重机制分析 ===\n");
+
+nodejieba.load();
+
+const content = "疯狂动物城 疯狂动物城 疯狂动物城 这是一个二次开发的项目，整合了原版的动画及Open Claw打包制作了MAC安装包，它可以出Open Claw现在你的系统桌面的任何地方，也会随互动有特定的动作，还蛮有意思的项目地址：https://github.com/justaLoli/VPet-Mac云盘：https://pan.quark.cn/s/62596470429a功能：✅开始、关闭、正常效果的动画播放✅拖动效果✅「互动」菜单里的互动，即睡觉、学习、工作等（带计时器，但没有经验、金钱加成）✅自动事件（发呆、待机、睡觉等）✅桌宠自动移动✅摸头预览";
+
+console.log("【问题分析】");
+console.log("关键词提取权重 = TF（词频）× IDF（逆文档频率）\n");
+
+console.log("步骤1: 查看分词结果");
+var cutResult = nodejieba.cut(content);
+console.log("分词结果:", cutResult.slice(0, 30));
+
+console.log("\n步骤2: 统计词频（TF）");
+var wordFreq = {};
+cutResult.forEach(word => {
+  wordFreq[word] = (wordFreq[word] || 0) + 1;
+});
+
+console.log("\n高频词（出现2次以上）:");
+Object.entries(wordFreq)
+  .filter(([word, freq]) => freq >= 2)
+  .sort((a, b) => b[1] - a[1])
+  .forEach(([word, freq]) => {
+    console.log(`  ${word}: ${freq}次`);
+  });
+
+console.log("\n步骤3: 提取关键词（未加载用户词典）");
+var extractResult1 = nodejieba.extract(content, 20);
+console.log("关键词及权重:");
+extractResult1.forEach((item, i) => {
+  console.log(`  ${i + 1}. ${item.word}: ${item.weight.toFixed(2)} (出现${wordFreq[item.word] || 1}次)`);
+});
+
+console.log("\n步骤4: 加载用户词典");
+nodejieba.loadUserDict("Open Claw 10 n");
+
+console.log("\n步骤5: 再次提取关键词");
+var extractResult2 = nodejieba.extract(content, 20);
+console.log("关键词及权重:");
+extractResult2.forEach((item, i) => {
+  console.log(`  ${i + 1}. ${item.word}: ${item.weight.toFixed(2)} (出现${wordFreq[item.word] || 1}次)`);
+});
+
+console.log("\n【核心问题】");
+console.log("1. 'Open Claw' 出现了 2 次（TF = 2）");
+console.log("2. '互动' 出现了 2 次（TF = 2）");
+console.log("3. 但 '互动' 的权重可能更高，因为：");
+console.log("   - '互动' 在 IDF 词典中有专门的权重值");
+console.log("   - 'Open Claw' 不在 IDF 词典中，使用平均 IDF 值");
+console.log("4. 如果 '互动' 的 IDF 值 > 平均 IDF 值，则权重更高");
+
+console.log("\n【解决方案】");
+console.log("需要为用户词典中的词设置 IDF 权重！");
+console.log("建议：在 loadUserDict 时，自动为用户词典中的词设置较高的 IDF 值");

package/diagnose_priority.js

@@ -0,0 +1,71 @@
+var nodejieba = require("./index.js");
+
+console.log("=== 问题诊断：Open Claw 未被识别 ===\n");
+
+console.log("【问题原因分析】");
+console.log("1. 用户词典必须在调用 extract() 之前加载");
+console.log("2. 用户词典会影响分词结果，进而影响关键词提取");
+console.log("3. 关键词提取依赖于分词结果和 IDF 权重\n");
+
+console.log("=== 测试场景 1: 错误用法（先提取后加载）===");
+nodejieba.load();
+const content = "这是一个Open Claw项目，Open Claw很好玩";
+
+console.log("步骤1: 先提取关键词（未加载用户词典）");
+var result1 = nodejieba.extract(content, 5);
+console.log("关键词:", result1.map(r => r.word));
+console.log("说明: 此时 'Open Claw' 被拆分成单个字母\n");
+
+console.log("步骤2: 再加载用户词典");
+nodejieba.loadUserDict("Open Claw 10 n");
+console.log("词典已加载\n");
+
+console.log("步骤3: 再次提取关键词");
+var result2 = nodejieba.extract(content, 5);
+console.log("关键词:", result2.map(r => r.word));
+console.log("说明: 此时 'Open Claw' 已被正确识别\n");
+
+console.log("=== 测试场景 2: 正确用法（先加载后提取）===");
+console.log("重新初始化...");
+delete require.cache[require.resolve('./index.js')];
+var nodejieba2 = require("./index.js");
+
+console.log("步骤1: 先加载词典");
+nodejieba2.load();
+nodejieba2.loadUserDict("Open Claw 10 n");
+console.log("词典已加载\n");
+
+console.log("步骤2: 再提取关键词");
+var result3 = nodejieba2.extract(content, 5);
+console.log("关键词:", result3.map(r => r.word));
+console.log("说明: 'Open Claw' 被正确识别\n");
+
+console.log("=== 测试场景 3: 检查优先级 ===");
+console.log("测试: 用户词典 vs 默认词典\n");
+
+var testCases = [
+  "Open Claw",
+  "Open Claw是一个项目",
+  "我喜欢Open Claw这个游戏"
+];
+
+testCases.forEach((text, i) => {
+  console.log(`测试 ${i + 1}: "${text}"`);
+  var cut = nodejieba2.cut(text);
+  var extract = nodejieba2.extract(text, 3);
+  console.log("  分词:", cut);
+  console.log("  关键词:", extract.map(r => r.word));
+  console.log("  包含 'Open Claw':", cut.includes("Open Claw"));
+  console.log();
+});
+
+console.log("=== 解决方案 ===");
+console.log("✅ 正确做法:");
+console.log("   1. 先调用 nodejieba.load()");
+console.log("   2. 再调用 nodejieba.loadUserDict('Open Claw 10 n')");
+console.log("   3. 最后调用 nodejieba.extract()");
+console.log();
+console.log("❌ 错误做法:");
+console.log("   1. 先调用 nodejieba.extract()");
+console.log("   2. 再调用 nodejieba.loadUserDict()");
+console.log("   （此时词典不会影响已提取的结果）");

package/index.js

@@ -74,6 +74,8 @@ wrapWithDictLoad("extract");
 wrapWithDictLoad("textRankExtract");
 wrapWithDictLoad("insertWord");
 wrapWithDictLoad("loadUserDict");
+wrapWithDictLoad("loadIdfDict");
+wrapWithDictLoad("setIdf");
 
 // 保存原始的 loadUserDict 函数
 var _loadUserDict = exports.loadUserDict;
@@ -99,4 +101,28 @@ exports.loadUserDict = function (dict) {
   return _loadUserDict.call(this, dict);
 };
 
+// 保存原始的 loadIdfDict 函数
+var _loadIdfDict = exports.loadIdfDict;
+
+// 重写 loadIdfDict 以支持 Set 格式
+exports.loadIdfDict = function (dict) {
+  if (!isDictLoaded) {
+    exports.load();
+  }
+  
+  if (dict === null || dict === undefined) {
+    return false;
+  }
+  
+  if (dict instanceof Set) {
+    dict = Array.from(dict);
+  }
+  
+  if (typeof dict !== 'string' && !Array.isArray(dict) && !Buffer.isBuffer(dict)) {
+    throw new TypeError('dict must be string, string[], Set<string>, or Buffer');
+  }
+  
+  return _loadIdfDict.call(this, dict);
+};
+
 module.exports = exports;

package/lib/nodejieba.cpp

@@ -21,7 +21,9 @@ NodeJieba::NodeJieba(Napi::Env env, Napi::Object exports) {
     InstanceMethod("extract", &NodeJieba::extract),
     InstanceMethod("textRankExtract", &NodeJieba::textRankExtract),
     InstanceMethod("insertWord", &NodeJieba::insertWord),
-    InstanceMethod("loadUserDict", &NodeJieba::loadUserDict)
+    InstanceMethod("loadUserDict", &NodeJieba::loadUserDict),
+    InstanceMethod("loadIdfDict", &NodeJieba::loadIdfDict),
+    InstanceMethod("setIdf", &NodeJieba::setIdf)
   });
 }
 
@@ -53,6 +55,216 @@ Napi::Value NodeJieba::load(const Napi::CallbackInfo& info) {
   return Napi::Boolean::New(info.Env(), true);
 }
 
+Napi::Value NodeJieba::loadIdfDict(const Napi::CallbackInfo& info) {
+  if (info.Length() < 1) {
+    return Napi::Boolean::New(info.Env(), false);
+  }
+
+  if( !_jieba_handle ){
+    Napi::Error::New(info.Env(), "Before calling any other function you have to call load() first").ThrowAsJavaScriptException();
+  }
+
+  auto isBlankString = [](const std::string& str) -> bool {
+    for (char c : str) {
+      if (!std::isspace(static_cast<unsigned char>(c))) {
+        return false;
+      }
+    }
+    return true;
+  };
+
+  auto trimString = [](std::string& str) -> void {
+    size_t start = 0;
+    size_t end = str.length();
+    
+    while (start < end && std::isspace(static_cast<unsigned char>(str[start]))) {
+      start++;
+    }
+    
+    while (end > start && std::isspace(static_cast<unsigned char>(str[end - 1]))) {
+      end--;
+    }
+    
+    str = str.substr(start, end - start);
+  };
+
+  auto toUpper = [](const std::string& str) -> std::string {
+    std::string result = str;
+    for (size_t i = 0; i < result.length(); i++) {
+      result[i] = std::toupper(static_cast<unsigned char>(result[i]));
+    }
+    return result;
+  };
+
+  auto toLower = [](const std::string& str) -> std::string {
+    std::string result = str;
+    for (size_t i = 0; i < result.length(); i++) {
+      result[i] = std::tolower(static_cast<unsigned char>(result[i]));
+    }
+    return result;
+  };
+
+  auto hasEnglishChars = [](const std::string& str) -> bool {
+    for (char c : str) {
+      if ((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z')) {
+        return true;
+      }
+    }
+    return false;
+  };
+
+  auto extractWordAndIdf = [](const std::string& line) -> std::pair<std::string, double> {
+    size_t lastSpace = line.find_last_of(" \t");
+    if (lastSpace == std::string::npos || lastSpace == 0) {
+      return std::make_pair("", 0.0);
+    }
+    
+    std::string word = line.substr(0, lastSpace);
+    std::string idfStr = line.substr(lastSpace + 1);
+    
+    char* endptr;
+    double idf = strtod(idfStr.c_str(), &endptr);
+    if (endptr == idfStr.c_str()) {
+      return std::make_pair("", 0.0);
+    }
+    
+    return std::make_pair(word, idf);
+  };
+
+  auto removeSpaces = [](const std::string& str) -> std::string {
+    std::string result;
+    for (char c : str) {
+      if (!std::isspace(static_cast<unsigned char>(c))) {
+        result += c;
+      }
+    }
+    return result;
+  };
+
+  auto processIdfLine = [&](const std::string& line) {
+    std::pair<std::string, double> parsed = extractWordAndIdf(line);
+    std::string word = parsed.first;
+    double idf = parsed.second;
+    
+    if (word.empty() || idf == 0.0) {
+      return;
+    }
+    
+    _jieba_handle->SetIdfForWord(word, idf);
+    _jieba_handle->InsertUserWord(word, "x");
+    
+    std::string wordNoSpace = removeSpaces(word);
+    if (wordNoSpace != word) {
+      _jieba_handle->SetIdfForWord(wordNoSpace, idf);
+      _jieba_handle->InsertUserWord(wordNoSpace, "x");
+    }
+    
+    if (hasEnglishChars(word)) {
+      std::string wordLower = toLower(word);
+      std::string wordUpper = toUpper(word);
+      
+      if (wordLower != word) {
+        _jieba_handle->SetIdfForWord(wordLower, idf);
+        _jieba_handle->InsertUserWord(wordLower, "x");
+      }
+      
+      if (wordUpper != word) {
+        _jieba_handle->SetIdfForWord(wordUpper, idf);
+        _jieba_handle->InsertUserWord(wordUpper, "x");
+      }
+      
+      std::string wordLowerNoSpace = toLower(wordNoSpace);
+      std::string wordUpperNoSpace = toUpper(wordNoSpace);
+      
+      if (wordLowerNoSpace != wordNoSpace && wordLowerNoSpace != wordLower) {
+        _jieba_handle->SetIdfForWord(wordLowerNoSpace, idf);
+        _jieba_handle->InsertUserWord(wordLowerNoSpace, "x");
+      }
+      
+      if (wordUpperNoSpace != wordNoSpace && wordUpperNoSpace != wordUpper) {
+        _jieba_handle->SetIdfForWord(wordUpperNoSpace, idf);
+        _jieba_handle->InsertUserWord(wordUpperNoSpace, "x");
+      }
+    }
+  };
+
+  if (info[0].IsArray()) {
+    Napi::Array arr = info[0].As<Napi::Array>();
+    for (size_t i = 0; i < arr.Length(); i++) {
+      Napi::Value val = arr[i];
+      if (!val.IsString()) {
+        Napi::TypeError::New(info.Env(), "Array elements must be strings")
+          .ThrowAsJavaScriptException();
+        return Napi::Boolean::New(info.Env(), false);
+      }
+      std::string line = val.As<Napi::String>().Utf8Value();
+      trimString(line);
+      if (!line.empty() && !isBlankString(line)) {
+        processIdfLine(line);
+      }
+    }
+  } else if (info[0].IsString()) {
+    std::string line = info[0].As<Napi::String>().Utf8Value();
+    trimString(line);
+    if (!line.empty() && !isBlankString(line)) {
+      processIdfLine(line);
+    }
+  } else if (info[0].IsBuffer()) {
+    Napi::Buffer<char> buffer = info[0].As<Napi::Buffer<char>>();
+    std::string content(buffer.Data(), buffer.Length());
+    std::istringstream iss(content);
+    std::string line;
+    while (std::getline(iss, line)) {
+      trimString(line);
+      if (!line.empty() && !isBlankString(line)) {
+        processIdfLine(line);
+      }
+    }
+  } else {
+    return Napi::Boolean::New(info.Env(), false);
+  }
+
+  return Napi::Boolean::New(info.Env(), true);
+}
+
+Napi::Value NodeJieba::setIdf(const Napi::CallbackInfo& info) {
+  if (info.Length() < 1) {
+    return Napi::Boolean::New(info.Env(), false);
+  }
+
+  if (!_jieba_handle) {
+    Napi::Error::New(info.Env(), "Before calling any other function you have to call load() first").ThrowAsJavaScriptException();
+    return Napi::Boolean::New(info.Env(), false);
+  }
+
+  std::string word;
+  double idf = 0.0;
+  double multiplier = 2.0;
+
+  if (info[0].IsString()) {
+    word = info[0].As<Napi::String>().Utf8Value();
+  } else {
+    return Napi::Boolean::New(info.Env(), false);
+  }
+
+  if (info.Length() >= 2) {
+    if (info[1].IsNumber()) {
+      idf = info[1].As<Napi::Number>().DoubleValue();
+      _jieba_handle->SetIdfForWord(word, idf);
+      return Napi::Boolean::New(info.Env(), true);
+    }
+  }
+
+  if (info.Length() >= 3) {
+    if (info[2].IsNumber()) {
+      multiplier = info[2].As<Napi::Number>().DoubleValue();
+    }
+  }
+
+  _jieba_handle->SetIdfForWordWithMultiplier(word, multiplier);
+  return Napi::Boolean::New(info.Env(), true);
+}
+
 Napi::Value NodeJieba::insertWord(const Napi::CallbackInfo& info) {
   if(info.Length() < 1) {
     return Napi::Boolean::New(info.Env(), false);
@@ -254,6 +466,70 @@ Napi::Value NodeJieba::loadUserDict(const Napi::CallbackInfo& info) {
     str = str.substr(start, end - start);
   };
 
+  auto extractKeyword = [](const std::string& line) -> std::string {
+    std::istringstream iss(line);
+    std::vector<std::string> parts;
+    std::string part;
+    
+    while (iss >> part) {
+      parts.push_back(part);
+    }
+    
+    if (parts.empty()) {
+      return "";
+    }
+    
+    if (parts.size() == 1) {
+      return parts[0];
+    }
+    
+    if (parts.size() == 2) {
+      size_t pos;
+      try {
+        std::stoi(parts[1], &pos);
+        if (pos == parts[1].length()) {
+          return parts[0];
+        }
+      } catch (...) {
+      }
+      return parts[0];
+    }
+    
+    if (parts.size() >= 3) {
+      size_t pos;
+      try {
+        std::stoi(parts[parts.size() - 2], &pos);
+        if (pos == parts[parts.size() - 2].length()) {
+          std::string keyword;
+          for (size_t i = 0; i < parts.size() - 2; i++) {
+            if (i > 0) keyword += " ";
+            keyword += parts[i];
+          }
+          return keyword;
+        }
+      } catch (...) {
+      }
+      
+      std::string keyword;
+      for (size_t i = 0; i < parts.size() - 1; i++) {
+        if (i > 0) keyword += " ";
+        keyword += parts[i];
+      }
+      return keyword;
+    }
+    
+    return parts[0];
+  };
+
+  auto setDefaultIdf = [&](const std::vector<std::string>& dictLines) {
+    for (const auto& line : dictLines) {
+      std::string keyword = extractKeyword(line);
+      if (!keyword.empty()) {
+        _jieba_handle->SetIdfForWordWithMultiplier(keyword, 2.0);
+      }
+    }
+  };
+
   if (info[0].IsArray()) {
     Napi::Array arr = info[0].As<Napi::Array>();
     std::vector<std::string> buf;
@@ -271,6 +547,7 @@ Napi::Value NodeJieba::loadUserDict(const Napi::CallbackInfo& info) {
       }
     }
     _jieba_handle->LoadUserDict(buf);
+    setDefaultIdf(buf);
   } else if (info[0].IsString()) {
     std::string line = info[0].As<Napi::String>().Utf8Value();
     trimString(line);
@@ -278,6 +555,7 @@ Napi::Value NodeJieba::loadUserDict(const Napi::CallbackInfo& info) {
     if (!line.empty() && !isBlankString(line)) {
       buf.push_back(line);
       _jieba_handle->LoadUserDict(buf);
+      setDefaultIdf(buf);
     }
   } else if (info[0].IsBuffer()) {
     Napi::Buffer<char> buffer = info[0].As<Napi::Buffer<char>>();
@@ -292,6 +570,7 @@ Napi::Value NodeJieba::loadUserDict(const Napi::CallbackInfo& info) {
       }
     }
     _jieba_handle->LoadUserDict(buf);
+    setDefaultIdf(buf);
   } else {
     return Napi::Boolean::New(info.Env(), false);
   }

package/lib/nodejieba.h

@@ -21,6 +21,8 @@ private:
   Napi::Value textRankExtract(const Napi::CallbackInfo& info);
   Napi::Value insertWord(const Napi::CallbackInfo& info);
   Napi::Value loadUserDict(const Napi::CallbackInfo& info);
+  Napi::Value loadIdfDict(const Napi::CallbackInfo& info);
+  Napi::Value setIdf(const Napi::CallbackInfo& info);
 
   cppjieba::Jieba* _jieba_handle{nullptr};
   cppjieba::TextRankExtractor* _text_rank_extractor_handle{nullptr};

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "nodejieba-plus",
   "description": "chinese word segmentation for node",
-  "version": "3.5.16",
+  "version": "3.5.18",
   "author": "Yanyi Wu <wuyanyi09@foxmail.com>",
   "maintainers": [
     "Yanyi Wu <wuyanyi09@foxmail.com>"

package/submodules/cppjieba/include/cppjieba/Jieba.hpp

@@ -116,6 +116,18 @@ class Jieba {
     dict_trie_.LoadUserDict(path);
   }
 
+  void SetIdfForWord(const string& word, double idf) {
+    extractor.SetIdf(word, idf);
+  }
+
+  void SetIdfForWordWithMultiplier(const string& word, double multiplier = 2.0) {
+    extractor.SetIdfWithMultiplier(word, multiplier);
+  }
+
+  void LoadIdfDict(const vector<string>& buf) {
+    extractor.LoadIdfDict(buf);
+  }
+
  private:
   static string pathJoin(const string& dir, const string& filename) {
     if (dir.empty()) {

package/submodules/cppjieba/include/cppjieba/KeywordExtractor.hpp

@@ -39,6 +39,54 @@ class KeywordExtractor {
   ~KeywordExtractor() {
   }
 
+  void SetIdf(const string& word, double idf) {
+    idfMap_[word] = idf;
+  }
+
+  void SetIdfWithMultiplier(const string& word, double multiplier = 2.0) {
+    unordered_map<string, double>::const_iterator cit = idfMap_.find(word);
+    if (cit != idfMap_.end()) {
+      idfMap_[word] = cit->second * multiplier;
+    } else {
+      idfMap_[word] = idfAverage_ * multiplier;
+    }
+  }
+
+  void LoadIdfDict(const vector<string>& buf) {
+    double idf = 0.0;
+    double idfSum = 0.0;
+    size_t validCount = 0;
+    
+    for (size_t i = 0; i < buf.size(); i++) {
+      const string& line = buf[i];
+      if (line.empty()) {
+        continue;
+      }
+      
+      size_t lastSpace = line.find_last_of(" \t");
+      if (lastSpace == string::npos) {
+        continue;
+      }
+      
+      string word = line.substr(0, lastSpace);
+      string idfStr = line.substr(lastSpace + 1);
+      
+      char* endptr;
+      idf = strtod(idfStr.c_str(), &endptr);
+      if (endptr == idfStr.c_str()) {
+        continue;
+      }
+      
+      idfMap_[word] = idf;
+      idfSum += idf;
+      validCount++;
+    }
+
+    if (validCount > 0) {
+      idfAverage_ = idfSum / validCount;
+    }
+  }
+
   void Extract(const string& sentence, vector<string>& keywords, size_t topN) const {
     vector<Word> topWords;
     Extract(sentence, topWords, topN);
@@ -96,25 +144,33 @@ class KeywordExtractor {
     ifstream ifs(idfPath.c_str());
     XCHECK(ifs.is_open()) << "open " << idfPath << " failed";
     string line ;
-    vector<string> buf;
     double idf = 0.0;
     double idfSum = 0.0;
     size_t lineno = 0;
     for (; getline(ifs, line); lineno++) {
-      buf.clear();
       if (line.empty()) {
         XLOG(ERROR) << "lineno: " << lineno << " empty. skipped.";
         continue;
       }
-      Split(line, buf, " ");
-      if (buf.size() != 2) {
-        XLOG(ERROR) << "line: " << line << ", lineno: " << lineno << " empty. skipped.";
+      
+      size_t lastSpace = line.find_last_of(" \t");
+      if (lastSpace == string::npos) {
+        XLOG(ERROR) << "line: " << line << ", lineno: " << lineno << " format error. skipped.";
         continue;
       }
-      idf = atof(buf[1].c_str());
-      idfMap_[buf[0]] = idf;
+      
+      string word = line.substr(0, lastSpace);
+      string idfStr = line.substr(lastSpace + 1);
+      
+      char* endptr;
+      idf = strtod(idfStr.c_str(), &endptr);
+      if (endptr == idfStr.c_str()) {
+        XLOG(ERROR) << "line: " << line << ", lineno: " << lineno << " idf format error. skipped.";
+        continue;
+      }
+      
+      idfMap_[word] = idf;
       idfSum += idf;
-
     }
 
     assert(lineno);

package/test_1_3x_weight.js

@@ -0,0 +1,86 @@
+var nodejieba = require("./index.js");
+var fs = require("fs");
+
+console.log("=== 测试 1.3 倍 IDF 权重 ===\n");
+
+const content = "疯狂动物城 疯狂动物城 疯狂动物城 这是一个二次开发的项目，整合了原版的动画及Open Claw打包制作了MAC安装包，它可以出Open Claw现在你的系统桌面的任何地方，也会随互动有特定的动作，还蛮有意思的项目地址：https://github.com/justaLoli/VPet-Mac云盘：https://pan.quark.cn/s/62596470429a功能：✅开始、关闭、正常效果的动画播放✅拖动效果✅「互动」菜单里的互动，即睡觉、学习、工作等（带计时器，但没有经验、金钱加成）✅自动事件（发呆、待机、睡觉等）✅桌宠自动移动✅摸头预览";
+
+console.log("【测试 1: 未加载用户词典】");
+nodejieba.load();
+var result1 = nodejieba.extract(content, 10);
+console.log("关键词及权重:");
+result1.forEach((item, i) => {
+  console.log(`  ${i + 1}. ${item.word}: ${item.weight.toFixed(2)}`);
+});
+
+console.log("\n【测试 2: 加载用户词典（自动 1.3 倍权重）】");
+nodejieba.loadUserDict("Open Claw 10 n");
+var result2 = nodejieba.extract(content, 10);
+console.log("关键词及权重:");
+result2.forEach((item, i) => {
+  console.log(`  ${i + 1}. ${item.word}: ${item.weight.toFixed(2)}`);
+});
+
+console.log("\n【验证 1.3 倍权重】");
+var openClawWeight1 = result1.find(r => r.word === "Open" || r.word === "Claw");
+var openClawWeight2 = result2.find(r => r.word === "Open Claw");
+if (openClawWeight1 && openClawWeight2) {
+  console.log(`未加载时权重: ${openClawWeight1.weight.toFixed(2)}`);
+  console.log(`加载后权重: ${openClawWeight2.weight.toFixed(2)}`);
+  console.log(`权重倍数: ${(openClawWeight2.weight / openClawWeight1.weight).toFixed(2)} ✅`);
+}
+
+console.log("\n=== 测试 IDF 词典支持空格关键词 ===\n");
+
+console.log("【测试 3: 创建包含空格关键词的 IDF 词典】");
+var testIdfPath = "./test_idf_with_spaces.txt";
+var testIdfContent = "Open Claw 30.0\nMachine Learning 25.0\nDeep Learning 28.0\n互动 12.0\n";
+fs.writeFileSync(testIdfPath, testIdfContent, "utf8");
+console.log("测试 IDF 词典内容:");
+console.log(testIdfContent);
+
+console.log("\n【测试 4: 加载包含空格的 IDF 词典 + 用户词典】");
+delete require.cache[require.resolve('./index.js')];
+var nodejieba2 = require("./index.js");
+
+nodejieba2.load({
+  idfDict: testIdfPath
+});
+
+nodejieba2.loadUserDict([
+  "Open Claw",
+  "Machine Learning",
+  "Deep Learning"
+]);
+
+console.log("\n【测试 5: 验证空格关键词识别】");
+var testContent = "Open Claw和Machine Learning都是Deep Learning的基础";
+var keywords = nodejieba2.extract(testContent, 5);
+console.log("测试文本:", testContent);
+console.log("关键词提取结果:");
+keywords.forEach((item, i) => {
+  console.log(`  ${i + 1}. ${item.word}: ${item.weight.toFixed(2)}`);
+});
+
+console.log("\n【验证结果】");
+var hasOpenClaw = keywords.some(r => r.word === "Open Claw");
+var hasMachineLearning = keywords.some(r => r.word === "Machine Learning");
+var hasDeepLearning = keywords.some(r => r.word === "Deep Learning");
+
+console.log(`识别到 'Open Claw': ${hasOpenClaw ? '✅' : '❌'}`);
+console.log(`识别到 'Machine Learning': ${hasMachineLearning ? '✅' : '❌'}`);
+console.log(`识别到 'Deep Learning': ${hasDeepLearning ? '✅' : '❌'}`);
+
+if (hasOpenClaw) {
+  var openClaw = keywords.find(r => r.word === "Open Claw");
+  console.log(`\n'Open Claw' 权重: ${openClaw.weight.toFixed(2)}`);
+  console.log(`IDF 词典中设置的权重: 30.0`);
+  console.log(`用户词典自动提升倍数: 1.3`);
+  console.log(`理论权重: ${(30.0 * 1.3).toFixed(2)} (30.0 × 1.3)`);
+}
+
+console.log("\n【清理测试文件】");
+fs.unlinkSync(testIdfPath);
+console.log("测试文件已删除");
+
+console.log("\n=== 测试完成 ===");

package/test_2x_weight.js

@@ -0,0 +1,39 @@
+var nodejieba = require("./index.js");
+
+console.log("=== 测试 2.0 倍 IDF 权重 ===\n");
+
+const content = "疯狂动物城 疯狂动物城 疯狂动物城 这是一个二次开发的项目，整合了原版的动画及Open Claw打包制作了MAC安装包，它可以出Open Claw现在你的系统桌面的任何地方，也会随互动有特定的动作，还蛮有意思的项目地址：https://github.com/justaLoli/VPet-Mac云盘：https://pan.quark.cn/s/62596470429a功能：✅开始、关闭、正常效果的动画播放✅拖动效果✅「互动」菜单里的互动，即睡觉、学习、工作等（带计时器，但没有经验、金钱加成）✅自动事件（发呆、待机、睡觉等）✅桌宠自动移动✅摸头预览";
+
+console.log("【测试 1: 未加载用户词典】");
+nodejieba.load();
+var result1 = nodejieba.extract(content, 10);
+console.log("关键词及权重:");
+result1.forEach((item, i) => {
+  console.log(`  ${i + 1}. ${item.word}: ${item.weight.toFixed(2)}`);
+});
+
+console.log("\n【测试 2: 加载用户词典（自动 2.0 倍权重）】");
+nodejieba.loadUserDict("Open Claw 10 n");
+var result2 = nodejieba.extract(content, 10);
+console.log("关键词及权重:");
+result2.forEach((item, i) => {
+  console.log(`  ${i + 1}. ${item.word}: ${item.weight.toFixed(2)}`);
+});
+
+console.log("\n【验证 2.0 倍权重】");
+var openClawWeight1 = result1.find(r => r.word === "Open" || r.word === "Claw");
+var openClawWeight2 = result2.find(r => r.word === "Open Claw");
+if (openClawWeight1 && openClawWeight2) {
+  console.log(`未加载时权重: ${openClawWeight1.weight.toFixed(2)}`);
+  console.log(`加载后权重: ${openClawWeight2.weight.toFixed(2)}`);
+  console.log(`权重倍数: ${(openClawWeight2.weight / openClawWeight1.weight).toFixed(2)}`);
+  
+  var ratio = openClawWeight2.weight / openClawWeight1.weight;
+  if (Math.abs(ratio - 2.0) < 0.01) {
+    console.log("✅ 2.0 倍权重验证成功！");
+  } else {
+    console.log(`❌ 权重倍数不符合预期（期望 2.0，实际 ${ratio.toFixed(2)}）`);
+  }
+}
+
+console.log("\n=== 测试完成 ===");

package/test_idf_feature.js

@@ -0,0 +1,43 @@
+var nodejieba = require("./index.js");
+
+console.log("=== 测试 IDF 权重提升功能 ===\n");
+
+const content = "疯狂动物城 疯狂动物城 疯狂动物城 这是一个二次开发的项目，整合了原版的动画及Open Claw打包制作了MAC安装包，它可以出Open Claw现在你的系统桌面的任何地方，也会随互动有特定的动作，还蛮有意思的项目地址：https://github.com/justaLoli/VPet-Mac云盘：https://pan.quark.cn/s/62596470429a功能：✅开始、关闭、正常效果的动画播放✅拖动效果✅「互动」菜单里的互动，即睡觉、学习、工作等（带计时器，但没有经验、金钱加成）✅自动事件（发呆、待机、睡觉等）✅桌宠自动移动✅摸头预览";
+
+console.log("【测试 1: 默认提取（未加载用户词典）】");
+nodejieba.load();
+var result1 = nodejieba.extract(content, 20);
+console.log("关键词及权重:");
+result1.forEach((item, i) => {
+  console.log(`  ${i + 1}. ${item.word}: ${item.weight.toFixed(2)}`);
+});
+
+console.log("\n【测试 2: 加载用户词典（自动提升 IDF 权重）】");
+nodejieba.loadUserDict("Open Claw 10 n");
+var result2 = nodejieba.extract(content, 20);
+console.log("关键词及权重:");
+result2.forEach((item, i) => {
+  console.log(`  ${i + 1}. ${item.word}: ${item.weight.toFixed(2)}`);
+});
+
+console.log("\n【测试 3: 手动设置 IDF 权重】");
+nodejieba.setIdf("Open Claw", 30.0);
+var result3 = nodejieba.extract(content, 20);
+console.log("关键词及权重:");
+result3.forEach((item, i) => {
+  console.log(`  ${i + 1}. ${item.word}: ${item.weight.toFixed(2)}`);
+});
+
+console.log("\n【测试 4: 使用倍数提升 IDF 权重】");
+nodejieba.setIdf("Open Claw", null, 3.0);  // 3倍权重
+var result4 = nodejieba.extract(content, 20);
+console.log("关键词及权重:");
+result4.forEach((item, i) => {
+  console.log(`  ${i + 1}. ${item.word}: ${item.weight.toFixed(2)}`);
+});
+
+console.log("\n【对比结果】");
+console.log("未加载词典时 'Open Claw' 排名:", result1.findIndex(r => r.word === "Open Claw") + 1 || "未出现");
+console.log("加载词典后 'Open Claw' 排名:", result2.findIndex(r => r.word === "Open Claw") + 1);
+console.log("手动设置IDF后 'Open Claw' 排名:", result3.findIndex(r => r.word === "Open Claw") + 1);
+console.log("3倍权重后 'Open Claw' 排名:", result4.findIndex(r => r.word === "Open Claw") + 1);

package/test_load_idf_dict.js

@@ -0,0 +1,98 @@
+var nodejieba = require('./index.js');
+
+console.log('=== 测试 loadIdfDict 功能 ===\n');
+
+// 初始化
+nodejieba.load();
+
+// 测试1: 使用数组加载IDF
+console.log('测试1: 使用数组加载IDF');
+var idfArray = [
+  '自定义词1 15.0',
+  '自定义词2 12.5',
+  '测试词汇 10.0'
+];
+var result1 = nodejieba.loadIdfDict(idfArray);
+console.log('数组加载结果:', result1);
+
+// 测试关键词提取
+var keywords1 = nodejieba.extract('这是一个自定义词1和自定义词2的测试词汇', 3);
+console.log('提取的关键词:', keywords1);
+console.log();
+
+// 测试2: 使用字符串加载单个IDF
+console.log('测试2: 使用字符串加载单个IDF');
+var result2 = nodejieba.loadIdfDict('单个词汇 20.0');
+console.log('字符串加载结果:', result2);
+
+var keywords2 = nodejieba.extract('这是一个单个词汇的测试', 3);
+console.log('提取的关键词:', keywords2);
+console.log();
+
+// 测试3: 使用Buffer加载IDF
+console.log('测试3: 使用Buffer加载IDF');
+var idfBuffer = Buffer.from('缓冲词汇1 18.0\n缓冲词汇2 16.5');
+var result3 = nodejieba.loadIdfDict(idfBuffer);
+console.log('Buffer加载结果:', result3);
+
+var keywords3 = nodejieba.extract('缓冲词汇1和缓冲词汇2的测试', 3);
+console.log('提取的关键词:', keywords3);
+console.log();
+
+// 测试4: 使用Set加载IDF
+console.log('测试4: 使用Set加载IDF');
+var idfSet = new Set(['集合词汇1 14.0', '集合词汇2 13.0']);
+var result4 = nodejieba.loadIdfDict(idfSet);
+console.log('Set加载结果:', result4);
+
+var keywords4 = nodejieba.extract('集合词汇1和集合词汇2的测试', 3);
+console.log('提取的关键词:', keywords4);
+console.log();
+
+// 测试5: 支持包含空格的词组
+console.log('测试5: 支持包含空格的词组');
+var idfWithSpace = ['人工智能 技术 25.0', '机器 学习 22.0'];
+var result5 = nodejieba.loadIdfDict(idfWithSpace);
+console.log('包含空格词组加载结果:', result5);
+
+// 测试提取包含空格的词组
+var keywords5a = nodejieba.extract('人工智能技术的发展和机器学习的应用', 5);
+console.log('提取的关键词(含空格):', keywords5a);
+
+// 测试不含空格的版本也能匹配
+var keywords5b = nodejieba.extract('人工智能技术发展和机器学习应用', 5);
+console.log('提取的关键词(不含空格):', keywords5b);
+console.log();
+
+// 测试6: 英文大小写不敏感
+console.log('测试6: 英文大小写不敏感');
+var idfEnglish = ['MachineLearning 30.0', 'Deep Learning 28.0', 'AI Technology 26.0'];
+var result6 = nodejieba.loadIdfDict(idfEnglish);
+console.log('英文IDF加载结果:', result6);
+
+// 测试大写匹配
+var keywords6a = nodejieba.extract('MachineLearning is important', 3);
+console.log('大写匹配:', keywords6a);
+
+// 测试小写匹配
+var keywords6b = nodejieba.extract('machinelearning is important', 3);
+console.log('小写匹配:', keywords6b);
+
+// 测试混合大小写匹配
+var keywords6c = nodejieba.extract('MACHINELEARNING is important', 3);
+console.log('全大写匹配:', keywords6c);
+
+// 测试包含空格的英文词组
+var keywords6d = nodejieba.extract('Deep Learning and AI Technology', 3);
+console.log('包含空格英文词组:', keywords6d);
+
+// 测试不含空格的英文词组
+var keywords6e = nodejieba.extract('DeepLearning and AITechnology', 3);
+console.log('不含空格英文词组:', keywords6e);
+
+// 测试小写版本的包含空格英文词组
+var keywords6f = nodejieba.extract('deep learning and ai technology', 3);
+console.log('小写包含空格英文词组:', keywords6f);
+console.log();
+
+console.log('=== 所有测试完成 ===');

package/test_open_claw.js

@@ -0,0 +1,65 @@
+var nodejieba = require("./index.js");
+
+const content = "疯狂动物城 疯狂动物城 疯狂动物城 这是一个二次开发的项目，整合了原版的动画及Open Claw打包制作了MAC安装包，它可以出Open Claw现在你的系统桌面的任何地方，也会随互动有特定的动作，还蛮有意思的项目地址：https://github.com/justaLoli/VPet-Mac云盘：https://pan.quark.cn/s/62596470429a功能：✅开始、关闭、正常效果的动画播放✅拖动效果✅「互动」菜单里的互动，即睡觉、学习、工作等（带计时器，但没有经验、金钱加成）✅自动事件（发呆、待机、睡觉等）✅桌宠自动移动✅摸头预览";
+
+console.log("=== 测试 1: 默认加载词典 ===");
+nodejieba.load();
+
+console.log("\n测试 1.1: 分词结果（未加载用户词典）");
+var cutResult1 = nodejieba.cut(content);
+console.log("分词结果:", cutResult1);
+console.log("包含 'Open Claw':", cutResult1.includes("Open Claw") || cutResult1.includes("Open") || cutResult1.includes("Claw"));
+
+console.log("\n测试 1.2: 关键词提取（未加载用户词典）");
+var extractResult1 = nodejieba.extract(content, 20);
+console.log("关键词提取结果:", extractResult1.map(r => r.word));
+
+console.log("\n=== 测试 2: 加载用户词典后 ===");
+console.log("加载用户词典: 'Open Claw 10 n'");
+var loadResult = nodejieba.loadUserDict("Open Claw 10 n");
+console.log("加载结果:", loadResult);
+
+console.log("\n测试 2.1: 分词结果（已加载用户词典）");
+var cutResult2 = nodejieba.cut(content);
+console.log("分词结果:", cutResult2);
+console.log("包含 'Open Claw':", cutResult2.includes("Open Claw"));
+
+console.log("\n测试 2.2: 关键词提取（已加载用户词典）");
+var extractResult2 = nodejieba.extract(content, 20);
+console.log("关键词提取结果:", extractResult2.map(r => r.word));
+console.log("包含 'Open Claw':", extractResult2.some(r => r.word === "Open Claw"));
+
+console.log("\n=== 测试 3: 使用 insertWord ===");
+nodejieba.insertWord("Open Claw", "n");
+console.log("插入词: 'Open Claw'");
+
+console.log("\n测试 3.1: 分词结果（使用 insertWord）");
+var cutResult3 = nodejieba.cut(content);
+console.log("分词结果:", cutResult3);
+console.log("包含 'Open Claw':", cutResult3.includes("Open Claw"));
+
+console.log("\n测试 3.2: 关键词提取（使用 insertWord）");
+var extractResult3 = nodejieba.extract(content, 20);
+console.log("关键词提取结果:", extractResult3.map(r => r.word));
+console.log("包含 'Open Claw':", extractResult3.some(r => r.word === "Open Claw"));
+
+console.log("\n=== 测试 4: 检查文本中的 Open Claw ===");
+var testText1 = "Open Claw是一个好游戏";
+var testText2 = "我喜欢Open Claw";
+var testText3 = "OpenClaw很好玩";
+
+console.log("\n测试文本1:", testText1);
+console.log("分词:", nodejieba.cut(testText1));
+console.log("关键词:", nodejieba.extract(testText1, 5).map(r => r.word));
+
+console.log("\n测试文本2:", testText2);
+console.log("分词:", nodejieba.cut(testText2));
+console.log("关键词:", nodejieba.extract(testText2, 5).map(r => r.word));
+
+console.log("\n测试文本3:", testText3);
+console.log("分词:", nodejieba.cut(testText3));
+console.log("关键词:", nodejieba.extract(testText3, 5).map(r => r.word));
+
+console.log("\n=== 测试 5: 检查 IDF 词典 ===");
+console.log("说明: 关键词提取需要 IDF 词典支持");
+console.log("用户词典只影响分词，不影响关键词提取的权重计算");

package/types/index.d.ts

@@ -28,4 +28,6 @@ declare module "nodejieba" {
   export function insertWord(word: string, tag?: string): boolean;
   export function cutSmall(sentence: string, small: number): string[];
   export function loadUserDict(dict: string | string[] | Set<string> | Buffer): boolean;
+  export function loadIdfDict(dict: string | string[] | Set<string> | Buffer): boolean;
+  export function setIdf(word: string, idf?: number, multiplier?: number): boolean;
 }