nodejieba-plus 3.5.8

package/submodules/cppjieba/include/cppjieba/HMMSegment.hpp

@@ -0,0 +1,190 @@
+#ifndef CPPJIBEA_HMMSEGMENT_H
+#define CPPJIBEA_HMMSEGMENT_H
+
+#include <iostream>
+#include <fstream>
+#include <memory.h>
+#include <cassert>
+#include "HMMModel.hpp"
+#include "SegmentBase.hpp"
+
+namespace cppjieba {
+class HMMSegment: public SegmentBase {
+ public:
+  HMMSegment(const string& filePath)
+  : model_(new HMMModel(filePath)), isNeedDestroy_(true) {
+  }
+  HMMSegment(const HMMModel* model) 
+  : model_(model), isNeedDestroy_(false) {
+  }
+  ~HMMSegment() {
+    if (isNeedDestroy_) {
+      delete model_;
+    }
+  }
+
+  void Cut(const string& sentence, 
+        vector<string>& words) const {
+    vector<Word> tmp;
+    Cut(sentence, tmp);
+    GetStringsFromWords(tmp, words);
+  }
+  void Cut(const string& sentence, 
+        vector<Word>& words) const {
+    PreFilter pre_filter(symbols_, sentence);
+    PreFilter::Range range;
+    vector<WordRange> wrs;
+    wrs.reserve(sentence.size()/2);
+    while (pre_filter.HasNext()) {
+      range = pre_filter.Next();
+      Cut(range.begin, range.end, wrs);
+    }
+    words.clear();
+    words.reserve(wrs.size());
+    GetWordsFromWordRanges(sentence, wrs, words);
+  }
+  void Cut(RuneStrArray::const_iterator begin, RuneStrArray::const_iterator end, vector<WordRange>& res) const {
+    RuneStrArray::const_iterator left = begin;
+    RuneStrArray::const_iterator right = begin;
+    while (right != end) {
+      if (right->rune < 0x80) {
+        if (left != right) {
+          InternalCut(left, right, res);
+        }
+        left = right;
+        do {
+          right = SequentialLetterRule(left, end);
+          if (right != left) {
+            break;
+          }
+          right = NumbersRule(left, end);
+          if (right != left) {
+            break;
+          }
+          right ++;
+        } while (false);
+        WordRange wr(left, right - 1);
+        res.push_back(wr);
+        left = right;
+      } else {
+        right++;
+      }
+    }
+    if (left != right) {
+      InternalCut(left, right, res);
+    }
+  }
+ private:
+  // sequential letters rule
+  RuneStrArray::const_iterator SequentialLetterRule(RuneStrArray::const_iterator begin, RuneStrArray::const_iterator end) const {
+    Rune x = begin->rune;
+    if (('a' <= x && x <= 'z') || ('A' <= x && x <= 'Z')) {
+      begin ++;
+    } else {
+      return begin;
+    }
+    while (begin != end) {
+      x = begin->rune;
+      if (('a' <= x && x <= 'z') || ('A' <= x && x <= 'Z') || ('0' <= x && x <= '9')) {
+        begin ++;
+      } else {
+        break;
+      }
+    }
+    return begin;
+  }
+  //
+  RuneStrArray::const_iterator NumbersRule(RuneStrArray::const_iterator begin, RuneStrArray::const_iterator end) const {
+    Rune x = begin->rune;
+    if ('0' <= x && x <= '9') {
+      begin ++;
+    } else {
+      return begin;
+    }
+    while (begin != end) {
+      x = begin->rune;
+      if ( ('0' <= x && x <= '9') || x == '.') {
+        begin++;
+      } else {
+        break;
+      }
+    }
+    return begin;
+  }
+  void InternalCut(RuneStrArray::const_iterator begin, RuneStrArray::const_iterator end, vector<WordRange>& res) const {
+    vector<size_t> status;
+    Viterbi(begin, end, status);
+
+    RuneStrArray::const_iterator left = begin;
+    RuneStrArray::const_iterator right;
+    for (size_t i = 0; i < status.size(); i++) {
+      if (status[i] % 2) { //if (HMMModel::E == status[i] || HMMModel::S == status[i])
+        right = begin + i + 1;
+        WordRange wr(left, right - 1);
+        res.push_back(wr);
+        left = right;
+      }
+    }
+  }
+
+  void Viterbi(RuneStrArray::const_iterator begin, 
+        RuneStrArray::const_iterator end, 
+        vector<size_t>& status) const {
+    size_t Y = HMMModel::STATUS_SUM;
+    size_t X = end - begin;
+
+    size_t XYSize = X * Y;
+    size_t now, old, stat;
+    double tmp, endE, endS;
+
+    vector<int> path(XYSize);
+    vector<double> weight(XYSize);
+
+    //start
+    for (size_t y = 0; y < Y; y++) {
+      weight[0 + y * X] = model_->startProb[y] + model_->GetEmitProb(model_->emitProbVec[y], begin->rune, MIN_DOUBLE);
+      path[0 + y * X] = -1;
+    }
+
+    double emitProb;
+
+    for (size_t x = 1; x < X; x++) {
+      for (size_t y = 0; y < Y; y++) {
+        now = x + y*X;
+        weight[now] = MIN_DOUBLE;
+        path[now] = HMMModel::E; // warning
+        emitProb = model_->GetEmitProb(model_->emitProbVec[y], (begin+x)->rune, MIN_DOUBLE);
+        for (size_t preY = 0; preY < Y; preY++) {
+          old = x - 1 + preY * X;
+          tmp = weight[old] + model_->transProb[preY][y] + emitProb;
+          if (tmp > weight[now]) {
+            weight[now] = tmp;
+            path[now] = preY;
+          }
+        }
+      }
+    }
+
+    endE = weight[X-1+HMMModel::E*X];
+    endS = weight[X-1+HMMModel::S*X];
+    stat = 0;
+    if (endE >= endS) {
+      stat = HMMModel::E;
+    } else {
+      stat = HMMModel::S;
+    }
+
+    status.resize(X);
+    for (int x = X -1 ; x >= 0; x--) {
+      status[x] = stat;
+      stat = path[x + stat*X];
+    }
+  }
+
+  const HMMModel* model_;
+  bool isNeedDestroy_;
+}; // class HMMSegment
+
+} // namespace cppjieba
+
+#endif

package/submodules/cppjieba/include/cppjieba/Jieba.hpp

@@ -0,0 +1,169 @@
+#ifndef CPPJIEAB_JIEBA_H
+#define CPPJIEAB_JIEBA_H
+
+#include "QuerySegment.hpp"
+#include "KeywordExtractor.hpp"
+
+namespace cppjieba {
+
+class Jieba {
+ public:
+  Jieba(const string& dict_path = "", 
+        const string& model_path = "",
+        const string& user_dict_path = "", 
+        const string& idf_path = "", 
+        const string& stop_word_path = "") 
+    : dict_trie_(getPath(dict_path, "jieba.dict.utf8"), getPath(user_dict_path, "user.dict.utf8")),
+      model_(getPath(model_path, "hmm_model.utf8")),
+      mp_seg_(&dict_trie_),
+      hmm_seg_(&model_),
+      mix_seg_(&dict_trie_, &model_),
+      full_seg_(&dict_trie_),
+      query_seg_(&dict_trie_, &model_),
+      extractor(&dict_trie_, &model_, 
+                getPath(idf_path, "idf.utf8"), 
+                getPath(stop_word_path, "stop_words.utf8")) {
+  }
+  ~Jieba() {
+  }
+
+  struct LocWord {
+    string word;
+    size_t begin;
+    size_t end;
+  }; // struct LocWord
+
+  void Cut(const string& sentence, vector<string>& words, bool hmm = true) const {
+    mix_seg_.Cut(sentence, words, hmm);
+  }
+  void Cut(const string& sentence, vector<Word>& words, bool hmm = true) const {
+    mix_seg_.Cut(sentence, words, hmm);
+  }
+  void CutAll(const string& sentence, vector<string>& words) const {
+    full_seg_.Cut(sentence, words);
+  }
+  void CutAll(const string& sentence, vector<Word>& words) const {
+    full_seg_.Cut(sentence, words);
+  }
+  void CutForSearch(const string& sentence, vector<string>& words, bool hmm = true) const {
+    query_seg_.Cut(sentence, words, hmm);
+  }
+  void CutForSearch(const string& sentence, vector<Word>& words, bool hmm = true) const {
+    query_seg_.Cut(sentence, words, hmm);
+  }
+  void CutHMM(const string& sentence, vector<string>& words) const {
+    hmm_seg_.Cut(sentence, words);
+  }
+  void CutHMM(const string& sentence, vector<Word>& words) const {
+    hmm_seg_.Cut(sentence, words);
+  }
+  void CutSmall(const string& sentence, vector<string>& words, size_t max_word_len) const {
+    mp_seg_.Cut(sentence, words, max_word_len);
+  }
+  void CutSmall(const string& sentence, vector<Word>& words, size_t max_word_len) const {
+    mp_seg_.Cut(sentence, words, max_word_len);
+  }
+  
+  void Tag(const string& sentence, vector<pair<string, string> >& words) const {
+    mix_seg_.Tag(sentence, words);
+  }
+  string LookupTag(const string &str) const {
+    return mix_seg_.LookupTag(str);
+  }
+  bool InsertUserWord(const string& word, const string& tag = UNKNOWN_TAG) {
+    return dict_trie_.InsertUserWord(word, tag);
+  }
+
+  bool InsertUserWord(const string& word,int freq, const string& tag = UNKNOWN_TAG) {
+    return dict_trie_.InsertUserWord(word,freq, tag);
+  }
+
+  bool DeleteUserWord(const string& word, const string& tag = UNKNOWN_TAG) {
+    return dict_trie_.DeleteUserWord(word, tag);
+  }
+  
+  bool Find(const string& word)
+  {
+    return dict_trie_.Find(word);
+  }
+
+  void ResetSeparators(const string& s) {
+    //TODO
+    mp_seg_.ResetSeparators(s);
+    hmm_seg_.ResetSeparators(s);
+    mix_seg_.ResetSeparators(s);
+    full_seg_.ResetSeparators(s);
+    query_seg_.ResetSeparators(s);
+  }
+
+  const DictTrie* GetDictTrie() const {
+    return &dict_trie_;
+  } 
+  
+  const HMMModel* GetHMMModel() const {
+    return &model_;
+  }
+
+  void LoadUserDict(const vector<string>& buf)  {
+    dict_trie_.LoadUserDict(buf);
+  }
+
+  void LoadUserDict(const set<string>& buf)  {
+    dict_trie_.LoadUserDict(buf);
+  }
+
+  void LoadUserDict(const string& path)  {
+    dict_trie_.LoadUserDict(path);
+  }
+
+ private:
+  static string pathJoin(const string& dir, const string& filename) {
+    if (dir.empty()) {
+        return filename;
+    }
+    
+    char last_char = dir[dir.length() - 1];
+    if (last_char == '/' || last_char == '\\') {
+        return dir + filename;
+    } else {
+        #ifdef _WIN32
+        return dir + '\\' + filename;
+        #else
+        return dir + '/' + filename;
+        #endif
+    }
+  }
+
+  static string getCurrentDirectory() {
+    string path(__FILE__);
+    size_t pos = path.find_last_of("/\\");
+    return (pos == string::npos) ? "" : path.substr(0, pos);
+  }
+
+  static string getPath(const string& path, const string& default_file) {
+    if (path.empty()) {
+      string current_dir = getCurrentDirectory();
+      string parent_dir = current_dir.substr(0, current_dir.find_last_of("/\\"));
+      string grandparent_dir = parent_dir.substr(0, parent_dir.find_last_of("/\\"));
+      return pathJoin(pathJoin(grandparent_dir, "dict"), default_file);
+    }
+    return path;
+  }
+
+  DictTrie dict_trie_;
+  HMMModel model_;
+  
+  // They share the same dict trie and model
+  MPSegment mp_seg_;
+  HMMSegment hmm_seg_;
+  MixSegment mix_seg_;
+  FullSegment full_seg_;
+  QuerySegment query_seg_;
+
+ public:
+  KeywordExtractor extractor;
+}; // class Jieba
+
+} // namespace cppjieba
+
+#endif // CPPJIEAB_JIEBA_H

package/submodules/cppjieba/include/cppjieba/KeywordExtractor.hpp

@@ -0,0 +1,153 @@
+#ifndef CPPJIEBA_KEYWORD_EXTRACTOR_H
+#define CPPJIEBA_KEYWORD_EXTRACTOR_H
+
+#include <cmath>
+#include <set>
+#include "MixSegment.hpp"
+
+namespace cppjieba {
+
+using namespace limonp;
+using namespace std;
+
+/*utf8*/
+class KeywordExtractor {
+ public:
+  struct Word {
+    string word;
+    vector<size_t> offsets;
+    double weight;
+  }; // struct Word
+
+  KeywordExtractor(const string& dictPath, 
+        const string& hmmFilePath, 
+        const string& idfPath, 
+        const string& stopWordPath, 
+        const string& userDict = "") 
+    : segment_(dictPath, hmmFilePath, userDict) {
+    LoadIdfDict(idfPath);
+    LoadStopWordDict(stopWordPath);
+  }
+  KeywordExtractor(const DictTrie* dictTrie, 
+        const HMMModel* model,
+        const string& idfPath, 
+        const string& stopWordPath) 
+    : segment_(dictTrie, model) {
+    LoadIdfDict(idfPath);
+    LoadStopWordDict(stopWordPath);
+  }
+  ~KeywordExtractor() {
+  }
+
+  void Extract(const string& sentence, vector<string>& keywords, size_t topN) const {
+    vector<Word> topWords;
+    Extract(sentence, topWords, topN);
+    for (size_t i = 0; i < topWords.size(); i++) {
+      keywords.push_back(topWords[i].word);
+    }
+  }
+
+  void Extract(const string& sentence, vector<pair<string, double> >& keywords, size_t topN) const {
+    vector<Word> topWords;
+    Extract(sentence, topWords, topN);
+    for (size_t i = 0; i < topWords.size(); i++) {
+      keywords.push_back(pair<string, double>(topWords[i].word, topWords[i].weight));
+    }
+  }
+
+  void Extract(const string& sentence, vector<Word>& keywords, size_t topN) const {
+    vector<string> words;
+    segment_.Cut(sentence, words);
+
+    map<string, Word> wordmap;
+    size_t offset = 0;
+    for (size_t i = 0; i < words.size(); ++i) {
+      size_t t = offset;
+      offset += words[i].size();
+      if (IsSingleWord(words[i]) || stopWords_.find(words[i]) != stopWords_.end()) {
+        continue;
+      }
+      wordmap[words[i]].offsets.push_back(t);
+      wordmap[words[i]].weight += 1.0;
+    }
+    if (offset != sentence.size()) {
+      XLOG(ERROR) << "words illegal";
+      return;
+    }
+
+    keywords.clear();
+    keywords.reserve(wordmap.size());
+    for (map<string, Word>::iterator itr = wordmap.begin(); itr != wordmap.end(); ++itr) {
+      unordered_map<string, double>::const_iterator cit = idfMap_.find(itr->first);
+      if (cit != idfMap_.end()) {
+        itr->second.weight *= cit->second;
+      } else {
+        itr->second.weight *= idfAverage_;
+      }
+      itr->second.word = itr->first;
+      keywords.push_back(itr->second);
+    }
+    topN = min(topN, keywords.size());
+    partial_sort(keywords.begin(), keywords.begin() + topN, keywords.end(), Compare);
+    keywords.resize(topN);
+  }
+ private:
+  void LoadIdfDict(const string& idfPath) {
+    ifstream ifs(idfPath.c_str());
+    XCHECK(ifs.is_open()) << "open " << idfPath << " failed";
+    string line ;
+    vector<string> buf;
+    double idf = 0.0;
+    double idfSum = 0.0;
+    size_t lineno = 0;
+    for (; getline(ifs, line); lineno++) {
+      buf.clear();
+      if (line.empty()) {
+        XLOG(ERROR) << "lineno: " << lineno << " empty. skipped.";
+        continue;
+      }
+      Split(line, buf, " ");
+      if (buf.size() != 2) {
+        XLOG(ERROR) << "line: " << line << ", lineno: " << lineno << " empty. skipped.";
+        continue;
+      }
+      idf = atof(buf[1].c_str());
+      idfMap_[buf[0]] = idf;
+      idfSum += idf;
+
+    }
+
+    assert(lineno);
+    idfAverage_ = idfSum / lineno;
+    assert(idfAverage_ > 0.0);
+  }
+  void LoadStopWordDict(const string& filePath) {
+    ifstream ifs(filePath.c_str());
+    XCHECK(ifs.is_open()) << "open " << filePath << " failed";
+    string line ;
+    while (getline(ifs, line)) {
+      stopWords_.insert(line);
+    }
+    assert(stopWords_.size());
+  }
+
+  static bool Compare(const Word& lhs, const Word& rhs) {
+    return lhs.weight > rhs.weight;
+  }
+
+  MixSegment segment_;
+  unordered_map<string, double> idfMap_;
+  double idfAverage_;
+
+  unordered_set<string> stopWords_;
+}; // class KeywordExtractor
+
+inline ostream& operator << (ostream& os, const KeywordExtractor::Word& word) {
+  return os << "{\"word\": \"" << word.word << "\", \"offset\": " << word.offsets << ", \"weight\": " << word.weight << "}"; 
+}
+
+} // namespace cppjieba
+
+#endif
+
+

package/submodules/cppjieba/include/cppjieba/MPSegment.hpp

@@ -0,0 +1,137 @@
+#ifndef CPPJIEBA_MPSEGMENT_H
+#define CPPJIEBA_MPSEGMENT_H
+
+#include <algorithm>
+#include <set>
+#include <cassert>
+#include "limonp/Logging.hpp"
+#include "DictTrie.hpp"
+#include "SegmentTagged.hpp"
+#include "PosTagger.hpp"
+
+namespace cppjieba {
+
+class MPSegment: public SegmentTagged {
+ public:
+  MPSegment(const string& dictPath, const string& userDictPath = "")
+    : dictTrie_(new DictTrie(dictPath, userDictPath)), isNeedDestroy_(true) {
+  }
+  MPSegment(const DictTrie* dictTrie)
+    : dictTrie_(dictTrie), isNeedDestroy_(false) {
+    assert(dictTrie_);
+  }
+  ~MPSegment() {
+    if (isNeedDestroy_) {
+      delete dictTrie_;
+    }
+  }
+
+  void Cut(const string& sentence, vector<string>& words) const {
+    Cut(sentence, words, MAX_WORD_LENGTH);
+  }
+
+  void Cut(const string& sentence,
+        vector<string>& words,
+        size_t max_word_len) const {
+    vector<Word> tmp;
+    Cut(sentence, tmp, max_word_len);
+    GetStringsFromWords(tmp, words);
+  }
+  void Cut(const string& sentence, 
+        vector<Word>& words, 
+        size_t max_word_len = MAX_WORD_LENGTH) const {
+    PreFilter pre_filter(symbols_, sentence);
+    PreFilter::Range range;
+    vector<WordRange> wrs;
+    wrs.reserve(sentence.size()/2);
+    while (pre_filter.HasNext()) {
+      range = pre_filter.Next();
+      Cut(range.begin, range.end, wrs, max_word_len);
+    }
+    words.clear();
+    words.reserve(wrs.size());
+    GetWordsFromWordRanges(sentence, wrs, words);
+  }
+  void Cut(RuneStrArray::const_iterator begin,
+           RuneStrArray::const_iterator end,
+           vector<WordRange>& words,
+           size_t max_word_len = MAX_WORD_LENGTH) const {
+    vector<Dag> dags;
+    dictTrie_->Find(begin, 
+          end, 
+          dags,
+          max_word_len);
+    CalcDP(dags);
+    CutByDag(begin, end, dags, words);
+  }
+
+  const DictTrie* GetDictTrie() const {
+    return dictTrie_;
+  }
+
+  bool Tag(const string& src, vector<pair<string, string> >& res) const {
+    return tagger_.Tag(src, res, *this);
+  }
+
+  bool IsUserDictSingleChineseWord(const Rune& value) const {
+    return dictTrie_->IsUserDictSingleChineseWord(value);
+  }
+ private:
+  void CalcDP(vector<Dag>& dags) const {
+    size_t nextPos;
+    const DictUnit* p;
+    double val;
+
+    for (vector<Dag>::reverse_iterator rit = dags.rbegin(); rit != dags.rend(); rit++) {
+      rit->pInfo = NULL;
+      rit->weight = MIN_DOUBLE;
+      assert(!rit->nexts.empty());
+      for (LocalVector<pair<size_t, const DictUnit*> >::const_iterator it = rit->nexts.begin(); it != rit->nexts.end(); it++) {
+        nextPos = it->first;
+        p = it->second;
+        val = 0.0;
+        if (nextPos + 1 < dags.size()) {
+          val += dags[nextPos + 1].weight;
+        }
+
+        if (p) {
+          val += p->weight;
+        } else {
+          val += dictTrie_->GetMinWeight();
+        }
+        if (val > rit->weight) {
+          rit->pInfo = p;
+          rit->weight = val;
+        }
+      }
+    }
+  }
+  void CutByDag(RuneStrArray::const_iterator begin, 
+        RuneStrArray::const_iterator end, 
+        const vector<Dag>& dags, 
+        vector<WordRange>& words) const {
+    size_t i = 0;
+    while (i < dags.size()) {
+      const DictUnit* p = dags[i].pInfo;
+      if (p) {
+        assert(p->word.size() >= 1);
+        WordRange wr(begin + i, begin + i + p->word.size() - 1);
+        words.push_back(wr);
+        i += p->word.size();
+      } else { //single chinese word
+        WordRange wr(begin + i, begin + i);
+        words.push_back(wr);
+        i++;
+      }
+    }
+  }
+
+  const DictTrie* dictTrie_;
+  bool isNeedDestroy_;
+  PosTagger tagger_;
+
+}; // class MPSegment
+
+} // namespace cppjieba
+
+#endif