nodejieba-plus 3.5.8

package/submodules/cppjieba/dict/user.dict.utf8

@@ -0,0 +1,4 @@
+云计算
+韩玉鉴赏
+蓝翔 nz
+区块链 10 nz

package/submodules/cppjieba/include/cppjieba/DictTrie.hpp

@@ -0,0 +1,381 @@
+#ifndef CPPJIEBA_DICT_TRIE_HPP
+#define CPPJIEBA_DICT_TRIE_HPP
+
+#include <iostream>
+#include <fstream>
+#include <map>
+#include <string>
+#include <cstring>
+#include <cstdlib>
+#include <stdint.h>
+#include <cmath>
+#include <limits>
+#include <algorithm>
+#include "limonp/StringUtil.hpp"
+#include "limonp/Logging.hpp"
+#include "Unicode.hpp"
+#include "Trie.hpp"
+
+namespace cppjieba {
+
+using namespace limonp;
+
+const double MIN_DOUBLE = -3.14e+100;
+const double MAX_DOUBLE = 3.14e+100;
+const size_t DICT_COLUMN_NUM = 3;
+const char* const UNKNOWN_TAG = "";
+
+class DictTrie {
+ public:
+  enum UserWordWeightOption {
+    WordWeightMin,
+    WordWeightMedian,
+    WordWeightMax,
+  }; // enum UserWordWeightOption
+
+  DictTrie(const string& dict_path, const string& user_dict_paths = "", UserWordWeightOption user_word_weight_opt = WordWeightMedian) {
+    Init(dict_path, user_dict_paths, user_word_weight_opt);
+  }
+
+  ~DictTrie() {
+    delete trie_;
+  }
+
+  bool InsertUserWord(const string& word, const string& tag = UNKNOWN_TAG) {
+    DictUnit node_info;
+    if (!MakeNodeInfo(node_info, word, user_word_default_weight_, tag)) {
+      return false;
+    }
+    active_node_infos_.push_back(node_info);
+    trie_->InsertNode(node_info.word, &active_node_infos_.back());
+    return true;
+  }
+
+  bool InsertUserWord(const string& word,int freq, const string& tag = UNKNOWN_TAG) {
+    DictUnit node_info;
+    double weight = freq ? log(1.0 * freq / freq_sum_) : user_word_default_weight_ ;
+    if (!MakeNodeInfo(node_info, word, weight , tag)) {
+      return false;
+    }
+    active_node_infos_.push_back(node_info);
+    trie_->InsertNode(node_info.word, &active_node_infos_.back());
+    return true;
+  }
+
+  bool DeleteUserWord(const string& word, const string& tag = UNKNOWN_TAG) {
+    DictUnit node_info;
+    if (!MakeNodeInfo(node_info, word, user_word_default_weight_, tag)) {
+      return false;
+    }
+    trie_->DeleteNode(node_info.word, &node_info);
+    return true;
+  }
+  
+  const DictUnit* Find(RuneStrArray::const_iterator begin, RuneStrArray::const_iterator end) const {
+    return trie_->Find(begin, end);
+  }
+
+  void Find(RuneStrArray::const_iterator begin, 
+        RuneStrArray::const_iterator end, 
+        vector<struct Dag>&res,
+        size_t max_word_len = MAX_WORD_LENGTH) const {
+    trie_->Find(begin, end, res, max_word_len);
+  }
+
+  bool Find(const string& word)
+  {
+    const DictUnit *tmp = NULL;
+    RuneStrArray runes;
+    if (!DecodeUTF8RunesInString(word, runes))
+    {
+      XLOG(ERROR) << "Decode failed.";
+    }
+    tmp = Find(runes.begin(), runes.end());
+    if (tmp == NULL)
+    {
+      return false;
+    }
+    else
+    {
+      return true;
+    }
+  }
+
+  bool IsUserDictSingleChineseWord(const Rune& word) const {
+    return IsIn(user_dict_single_chinese_word_, word);
+  }
+
+  double GetMinWeight() const {
+    return min_weight_;
+  }
+
+  void InserUserDictNode(const string& line) {
+    vector<string> buf;
+    DictUnit node_info;
+    Split(line, buf, " ");
+    
+    string word;
+    string tag = UNKNOWN_TAG;
+    double weight = user_word_default_weight_;
+    bool hasSpace = false;
+    
+    // 处理包含空格的关键词
+    // 格式可能是: "word" 或 "word tag" 或 "word freq tag"
+    // 其中 word 本身可能包含空格
+    if (buf.size() == 1) {
+      // 只有关键词，无词频和标签
+      word = buf[0];
+    } else if (buf.size() == 2) {
+      // 可能是 "word tag" 或 "word1 word2"
+      // 检查第二个字段是否为数字（词频）
+      bool isNumber = true;
+      for (char c : buf[1]) {
+        if (!isdigit(c)) {
+          isNumber = false;
+          break;
+        }
+      }
+      if (isNumber) {
+        // "word freq" 格式，无标签
+        int freq = atoi(buf[1].c_str());
+        assert(freq_sum_ > 0.0);
+        weight = log(1.0 * freq / freq_sum_);
+        word = buf[0];
+      } else {
+        // "word tag" 格式
+        word = buf[0];
+        tag = buf[1];
+      }
+    } else {
+      // 检查最后两个字段：可能是 "... word freq tag" 或 "... word1 word2 tag" 等
+      // 倒数第二个如果是数字，则认为是词频，最后一个是标签
+      // 否则认为只有最后一个是标签，前面都是关键词
+      bool isFreq = true;
+      for (char c : buf[buf.size() - 2]) {
+        if (!isdigit(c)) {
+          isFreq = false;
+          break;
+        }
+      }
+      
+      if (isFreq) {
+        // 格式: "word... freq tag"
+        int freq = atoi(buf[buf.size() - 2].c_str());
+        assert(freq_sum_ > 0.0);
+        weight = log(1.0 * freq / freq_sum_);
+        // 前面的所有部分（除了最后两个）组成关键词
+        for (size_t i = 0; i < buf.size() - 2; ++i) {
+          if (i > 0) word += " ";
+          word += buf[i];
+        }
+        tag = buf[buf.size() - 1];
+      } else {
+        // 格式: "word... tag" (无词频)
+        // 前面的所有部分（除了最后一个）组成关键词
+        for (size_t i = 0; i < buf.size() - 1; ++i) {
+          if (i > 0) word += " ";
+          word += buf[i];
+        }
+        tag = buf[buf.size() - 1];
+      }
+    }
+    
+    // 检查词中是否包含空格
+    hasSpace = (word.find(' ') != string::npos);
+    
+    // 添加原始词（包含空格版本）
+    MakeNodeInfo(node_info, word, weight, tag);
+    static_node_infos_.push_back(node_info);
+    if (node_info.word.size() == 1) {
+      user_dict_single_chinese_word_.insert(node_info.word[0]);
+    }
+    
+    // 如果词包含空格，同时添加无空格版本
+    if (hasSpace) {
+      string wordNoSpace = word;
+      // 移除所有空格
+      wordNoSpace.erase(remove(wordNoSpace.begin(), wordNoSpace.end(), ' '), wordNoSpace.end());
+      if (!wordNoSpace.empty() && wordNoSpace != word) {
+        DictUnit node_info_no_space;
+        MakeNodeInfo(node_info_no_space, wordNoSpace, weight, tag);
+        static_node_infos_.push_back(node_info_no_space);
+        if (node_info_no_space.word.size() == 1) {
+          user_dict_single_chinese_word_.insert(node_info_no_space.word[0]);
+        }
+      }
+    }
+  }
+  
+  void LoadUserDict(const vector<string>& buf) {
+    for (size_t i = 0; i < buf.size(); i++) {
+      InserUserDictNode(buf[i]);
+    }
+  }
+
+   void LoadUserDict(const set<string>& buf) {
+    std::set<string>::const_iterator iter;
+    for (iter = buf.begin(); iter != buf.end(); iter++){
+      InserUserDictNode(*iter);
+    }
+  }
+
+  void LoadUserDict(const string& filePaths) {
+    vector<string> files = limonp::Split(filePaths, "|;");
+    for (size_t i = 0; i < files.size(); i++) {
+      ifstream ifs(files[i].c_str());
+      XCHECK(ifs.is_open()) << "open " << files[i] << " failed"; 
+      string line;
+      
+      while(getline(ifs, line)) {
+        if (line.size() == 0) {
+          continue;
+        }
+        InserUserDictNode(line);
+      }
+    }
+  }
+
+
+ private:
+  void Init(const string& dict_path, const string& user_dict_paths, UserWordWeightOption user_word_weight_opt) {
+    LoadDict(dict_path);
+    freq_sum_ = CalcFreqSum(static_node_infos_);
+    CalculateWeight(static_node_infos_, freq_sum_);
+    SetStaticWordWeights(user_word_weight_opt);
+
+    if (user_dict_paths.size()) {
+      LoadUserDict(user_dict_paths);
+    }
+    Shrink(static_node_infos_);
+    CreateTrie(static_node_infos_);
+  }
+  
+  void CreateTrie(const vector<DictUnit>& dictUnits) {
+    assert(dictUnits.size());
+    vector<Unicode> words;
+    vector<const DictUnit*> valuePointers;
+    for (size_t i = 0 ; i < dictUnits.size(); i ++) {
+      words.push_back(dictUnits[i].word);
+      valuePointers.push_back(&dictUnits[i]);
+    }
+
+    trie_ = new Trie(words, valuePointers);
+  }
+
+  
+
+
+  bool MakeNodeInfo(DictUnit& node_info,
+        const string& word, 
+        double weight, 
+        const string& tag) {
+    if (!DecodeUTF8RunesInString(word, node_info.word)) {
+      XLOG(ERROR) << "UTF-8 decode failed for dict word: " << word;
+      return false;
+    }
+    node_info.weight = weight;
+    node_info.tag = tag;
+    return true;
+  }
+
+  void LoadDict(const string& filePath) {
+    ifstream ifs(filePath.c_str());
+    XCHECK(ifs.is_open()) << "open " << filePath << " failed.";
+    string line;
+    vector<string> buf;
+
+    DictUnit node_info;
+    while (getline(ifs, line)) {
+      Split(line, buf, " ");
+      // 支持包含空格的关键词
+      // 格式: "word weight tag" 或 "word1 word2 weight tag" 等
+      // 最后两个字段是 weight 和 tag，前面的都是关键词
+      if (buf.size() >= DICT_COLUMN_NUM) {
+        // 组合前面的字段作为关键词
+        string word;
+        for (size_t i = 0; i < buf.size() - 2; ++i) {
+          if (i > 0) word += " ";
+          word += buf[i];
+        }
+        double weight = atof(buf[buf.size() - 2].c_str());
+        string tag = buf[buf.size() - 1];
+        
+        // 添加原始词（包含空格版本）
+        MakeNodeInfo(node_info, word, weight, tag);
+        static_node_infos_.push_back(node_info);
+        
+        // 如果词包含空格，同时添加无空格版本
+        if (word.find(' ') != string::npos) {
+          string wordNoSpace = word;
+          wordNoSpace.erase(remove(wordNoSpace.begin(), wordNoSpace.end(), ' '), wordNoSpace.end());
+          if (!wordNoSpace.empty() && wordNoSpace != word) {
+            DictUnit node_info_no_space;
+            MakeNodeInfo(node_info_no_space, wordNoSpace, weight, tag);
+            static_node_infos_.push_back(node_info_no_space);
+          }
+        }
+      } else {
+        XCHECK(buf.size() == DICT_COLUMN_NUM) << "split result illegal, line:" << line;
+      }
+    }
+  }
+
+  static bool WeightCompare(const DictUnit& lhs, const DictUnit& rhs) {
+    return lhs.weight < rhs.weight;
+  }
+
+  void SetStaticWordWeights(UserWordWeightOption option) {
+    XCHECK(!static_node_infos_.empty());
+    vector<DictUnit> x = static_node_infos_;
+    sort(x.begin(), x.end(), WeightCompare);
+    min_weight_ = x[0].weight;
+    max_weight_ = x[x.size() - 1].weight;
+    median_weight_ = x[x.size() / 2].weight;
+    switch (option) {
+     case WordWeightMin:
+       user_word_default_weight_ = min_weight_;
+       break;
+     case WordWeightMedian:
+       user_word_default_weight_ = median_weight_;
+       break;
+     default:
+       user_word_default_weight_ = max_weight_;
+       break;
+    }
+  }
+
+  double CalcFreqSum(const vector<DictUnit>& node_infos) const {
+    double sum = 0.0;
+    for (size_t i = 0; i < node_infos.size(); i++) {
+      sum += node_infos[i].weight;
+    }
+    return sum;
+  }
+
+  void CalculateWeight(vector<DictUnit>& node_infos, double sum) const {
+    assert(sum > 0.0);
+    for (size_t i = 0; i < node_infos.size(); i++) {
+      DictUnit& node_info = node_infos[i];
+      assert(node_info.weight > 0.0);
+      node_info.weight = log(double(node_info.weight)/sum);
+    }
+  }
+
+  void Shrink(vector<DictUnit>& units) const {
+    vector<DictUnit>(units.begin(), units.end()).swap(units);
+  }
+
+  vector<DictUnit> static_node_infos_;
+  deque<DictUnit> active_node_infos_; // must not be vector
+  Trie * trie_;
+
+  double freq_sum_;
+  double min_weight_;
+  double max_weight_;
+  double median_weight_;
+  double user_word_default_weight_;
+  unordered_set<Rune> user_dict_single_chinese_word_;
+};
+}
+
+#endif

package/submodules/cppjieba/include/cppjieba/FullSegment.hpp

@@ -0,0 +1,93 @@
+#ifndef CPPJIEBA_FULLSEGMENT_H
+#define CPPJIEBA_FULLSEGMENT_H
+
+#include <algorithm>
+#include <set>
+#include <cassert>
+#include "limonp/Logging.hpp"
+#include "DictTrie.hpp"
+#include "SegmentBase.hpp"
+#include "Unicode.hpp"
+
+namespace cppjieba {
+class FullSegment: public SegmentBase {
+ public:
+  FullSegment(const string& dictPath) {
+    dictTrie_ = new DictTrie(dictPath);
+    isNeedDestroy_ = true;
+  }
+  FullSegment(const DictTrie* dictTrie)
+    : dictTrie_(dictTrie), isNeedDestroy_(false) {
+    assert(dictTrie_);
+  }
+  ~FullSegment() {
+    if (isNeedDestroy_) {
+      delete dictTrie_;
+    }
+  }
+  void Cut(const string& sentence, 
+        vector<string>& words) const {
+    vector<Word> tmp;
+    Cut(sentence, tmp);
+    GetStringsFromWords(tmp, words);
+  }
+  void Cut(const string& sentence, 
+        vector<Word>& words) const {
+    PreFilter pre_filter(symbols_, sentence);
+    PreFilter::Range range;
+    vector<WordRange> wrs;
+    wrs.reserve(sentence.size()/2);
+    while (pre_filter.HasNext()) {
+      range = pre_filter.Next();
+      Cut(range.begin, range.end, wrs);
+    }
+    words.clear();
+    words.reserve(wrs.size());
+    GetWordsFromWordRanges(sentence, wrs, words);
+  }
+  void Cut(RuneStrArray::const_iterator begin, 
+        RuneStrArray::const_iterator end, 
+        vector<WordRange>& res) const {
+    // result of searching in trie tree
+    LocalVector<pair<size_t, const DictUnit*> > tRes;
+
+    // max index of res's words
+    size_t maxIdx = 0;
+
+    // always equals to (uItr - begin)
+    size_t uIdx = 0;
+
+    // tmp variables
+    size_t wordLen = 0;
+    assert(dictTrie_);
+    vector<struct Dag> dags;
+    dictTrie_->Find(begin, end, dags);
+    for (size_t i = 0; i < dags.size(); i++) {
+      for (size_t j = 0; j < dags[i].nexts.size(); j++) {
+        size_t nextoffset = dags[i].nexts[j].first;
+        assert(nextoffset < dags.size());
+        const DictUnit* du = dags[i].nexts[j].second;
+        if (du == NULL) {
+          if (dags[i].nexts.size() == 1 && maxIdx <= uIdx) {
+            WordRange wr(begin + i, begin + nextoffset);
+            res.push_back(wr);
+          }
+        } else {
+          wordLen = du->word.size();
+          if (wordLen >= 2 || (dags[i].nexts.size() == 1 && maxIdx <= uIdx)) {
+            WordRange wr(begin + i, begin + nextoffset);
+            res.push_back(wr);
+          }
+        }
+        maxIdx = uIdx + wordLen > maxIdx ? uIdx + wordLen : maxIdx;
+      }
+      uIdx++;
+    }
+  }
+ private:
+  const DictTrie* dictTrie_;
+  bool isNeedDestroy_;
+};
+}
+
+#endif

package/submodules/cppjieba/include/cppjieba/HMMModel.hpp

@@ -0,0 +1,129 @@
+#ifndef CPPJIEBA_HMMMODEL_H
+#define CPPJIEBA_HMMMODEL_H
+
+#include "limonp/StringUtil.hpp"
+#include "Trie.hpp"
+
+namespace cppjieba {
+
+using namespace limonp;
+typedef unordered_map<Rune, double> EmitProbMap;
+
+struct HMMModel {
+  /*
+   * STATUS:
+   * 0: HMMModel::B, 1: HMMModel::E, 2: HMMModel::M, 3:HMMModel::S
+   * */
+  enum {B = 0, E = 1, M = 2, S = 3, STATUS_SUM = 4};
+
+  HMMModel(const string& modelPath) {
+    memset(startProb, 0, sizeof(startProb));
+    memset(transProb, 0, sizeof(transProb));
+    statMap[0] = 'B';
+    statMap[1] = 'E';
+    statMap[2] = 'M';
+    statMap[3] = 'S';
+    emitProbVec.push_back(&emitProbB);
+    emitProbVec.push_back(&emitProbE);
+    emitProbVec.push_back(&emitProbM);
+    emitProbVec.push_back(&emitProbS);
+    LoadModel(modelPath);
+  }
+  ~HMMModel() {
+  }
+  void LoadModel(const string& filePath) {
+    ifstream ifile(filePath.c_str());
+    XCHECK(ifile.is_open()) << "open " << filePath << " failed";
+    string line;
+    vector<string> tmp;
+    vector<string> tmp2;
+    //Load startProb
+    XCHECK(GetLine(ifile, line));
+    Split(line, tmp, " ");
+    XCHECK(tmp.size() == STATUS_SUM);
+    for (size_t j = 0; j< tmp.size(); j++) {
+      startProb[j] = atof(tmp[j].c_str());
+    }
+
+    //Load transProb
+    for (size_t i = 0; i < STATUS_SUM; i++) {
+      XCHECK(GetLine(ifile, line));
+      Split(line, tmp, " ");
+      XCHECK(tmp.size() == STATUS_SUM);
+      for (size_t j =0; j < STATUS_SUM; j++) {
+        transProb[i][j] = atof(tmp[j].c_str());
+      }
+    }
+
+    //Load emitProbB
+    XCHECK(GetLine(ifile, line));
+    XCHECK(LoadEmitProb(line, emitProbB));
+
+    //Load emitProbE
+    XCHECK(GetLine(ifile, line));
+    XCHECK(LoadEmitProb(line, emitProbE));
+
+    //Load emitProbM
+    XCHECK(GetLine(ifile, line));
+    XCHECK(LoadEmitProb(line, emitProbM));
+
+    //Load emitProbS
+    XCHECK(GetLine(ifile, line));
+    XCHECK(LoadEmitProb(line, emitProbS));
+  }
+  double GetEmitProb(const EmitProbMap* ptMp, Rune key, 
+        double defVal)const {
+    EmitProbMap::const_iterator cit = ptMp->find(key);
+    if (cit == ptMp->end()) {
+      return defVal;
+    }
+    return cit->second;
+  }
+  bool GetLine(ifstream& ifile, string& line) {
+    while (getline(ifile, line)) {
+      Trim(line);
+      if (line.empty()) {
+        continue;
+      }
+      if (StartsWith(line, "#")) {
+        continue;
+      }
+      return true;
+    }
+    return false;
+  }
+  bool LoadEmitProb(const string& line, EmitProbMap& mp) {
+    if (line.empty()) {
+      return false;
+    }
+    vector<string> tmp, tmp2;
+    Unicode unicode;
+    Split(line, tmp, ",");
+    for (size_t i = 0; i < tmp.size(); i++) {
+      Split(tmp[i], tmp2, ":");
+      if (2 != tmp2.size()) {
+        XLOG(ERROR) << "emitProb illegal.";
+        return false;
+      }
+      if (!DecodeUTF8RunesInString(tmp2[0], unicode) || unicode.size() != 1) {
+        XLOG(ERROR) << "TransCode failed.";
+        return false;
+      }
+      mp[unicode[0]] = atof(tmp2[1].c_str());
+    }
+    return true;
+  }
+
+  char statMap[STATUS_SUM];
+  double startProb[STATUS_SUM];
+  double transProb[STATUS_SUM][STATUS_SUM];
+  EmitProbMap emitProbB;
+  EmitProbMap emitProbE;
+  EmitProbMap emitProbM;
+  EmitProbMap emitProbS;
+  vector<EmitProbMap* > emitProbVec;
+}; // struct HMMModel
+
+} // namespace cppjieba
+
+#endif