nodejieba-plus 3.5.8

package/submodules/cppjieba/include/cppjieba/MixSegment.hpp

@@ -0,0 +1,109 @@
+#ifndef CPPJIEBA_MIXSEGMENT_H
+#define CPPJIEBA_MIXSEGMENT_H
+
+#include <cassert>
+#include "MPSegment.hpp"
+#include "HMMSegment.hpp"
+#include "limonp/StringUtil.hpp"
+#include "PosTagger.hpp"
+
+namespace cppjieba {
+class MixSegment: public SegmentTagged {
+ public:
+  MixSegment(const string& mpSegDict, const string& hmmSegDict, 
+        const string& userDict = "") 
+    : mpSeg_(mpSegDict, userDict), 
+      hmmSeg_(hmmSegDict) {
+  }
+  MixSegment(const DictTrie* dictTrie, const HMMModel* model) 
+    : mpSeg_(dictTrie), hmmSeg_(model) {
+  }
+  ~MixSegment() {
+  }
+
+  void Cut(const string& sentence, vector<string>& words) const {
+    Cut(sentence, words, true);
+  }
+  void Cut(const string& sentence, vector<string>& words, bool hmm) const {
+    vector<Word> tmp;
+    Cut(sentence, tmp, hmm);
+    GetStringsFromWords(tmp, words);
+  }
+  void Cut(const string& sentence, vector<Word>& words, bool hmm = true) const {
+    PreFilter pre_filter(symbols_, sentence);
+    PreFilter::Range range;
+    vector<WordRange> wrs;
+    wrs.reserve(sentence.size() / 2);
+    while (pre_filter.HasNext()) {
+      range = pre_filter.Next();
+      Cut(range.begin, range.end, wrs, hmm);
+    }
+    words.clear();
+    words.reserve(wrs.size());
+    GetWordsFromWordRanges(sentence, wrs, words);
+  }
+
+  void Cut(RuneStrArray::const_iterator begin, RuneStrArray::const_iterator end, vector<WordRange>& res, bool hmm) const {
+    if (!hmm) {
+      mpSeg_.Cut(begin, end, res);
+      return;
+    }
+    vector<WordRange> words;
+    assert(end >= begin);
+    words.reserve(end - begin);
+    mpSeg_.Cut(begin, end, words);
+
+    vector<WordRange> hmmRes;
+    hmmRes.reserve(end - begin);
+    for (size_t i = 0; i < words.size(); i++) {
+      //if mp Get a word, it's ok, put it into result
+      if (words[i].left != words[i].right || (words[i].left == words[i].right && mpSeg_.IsUserDictSingleChineseWord(words[i].left->rune))) {
+        res.push_back(words[i]);
+        continue;
+      }
+
+      // if mp Get a single one and it is not in userdict, collect it in sequence
+      size_t j = i;
+      while (j < words.size() && words[j].left == words[j].right && !mpSeg_.IsUserDictSingleChineseWord(words[j].left->rune)) {
+        j++;
+      }
+
+      // Cut the sequence with hmm
+      assert(j - 1 >= i);
+      // TODO
+      hmmSeg_.Cut(words[i].left, words[j - 1].left + 1, hmmRes);
+      //put hmm result to result
+      for (size_t k = 0; k < hmmRes.size(); k++) {
+        res.push_back(hmmRes[k]);
+      }
+
+      //clear tmp vars
+      hmmRes.clear();
+
+      //let i jump over this piece
+      i = j - 1;
+    }
+  }
+
+  const DictTrie* GetDictTrie() const {
+    return mpSeg_.GetDictTrie();
+  }
+
+  bool Tag(const string& src, vector<pair<string, string> >& res) const {
+    return tagger_.Tag(src, res, *this);
+  }
+
+  string LookupTag(const string &str) const {
+    return tagger_.LookupTag(str, *this);
+  }
+
+ private:
+  MPSegment mpSeg_;
+  HMMSegment hmmSeg_;
+  PosTagger tagger_;
+
+}; // class MixSegment
+
+} // namespace cppjieba
+
+#endif

package/submodules/cppjieba/include/cppjieba/PosTagger.hpp

@@ -0,0 +1,77 @@
+#ifndef CPPJIEBA_POS_TAGGING_H
+#define CPPJIEBA_POS_TAGGING_H
+
+#include "limonp/StringUtil.hpp"
+#include "SegmentTagged.hpp"
+#include "DictTrie.hpp"
+
+namespace cppjieba {
+using namespace limonp;
+
+static const char* const POS_M = "m";
+static const char* const POS_ENG = "eng";
+static const char* const POS_X = "x";
+
+class PosTagger {
+ public:
+  PosTagger() {
+  }
+  ~PosTagger() {
+  }
+
+  bool Tag(const string& src, vector<pair<string, string> >& res, const SegmentTagged& segment) const {
+    vector<string> CutRes;
+    segment.Cut(src, CutRes);
+
+    for (vector<string>::iterator itr = CutRes.begin(); itr != CutRes.end(); ++itr) {
+      res.push_back(make_pair(*itr, LookupTag(*itr, segment)));
+    }
+    return !res.empty();
+  }
+
+  string LookupTag(const string &str, const SegmentTagged& segment) const {
+    const DictUnit *tmp = NULL;
+    RuneStrArray runes;
+    const DictTrie * dict = segment.GetDictTrie();
+    assert(dict != NULL);
+      if (!DecodeUTF8RunesInString(str, runes)) {
+        XLOG(ERROR) << "UTF-8 decode failed for word: " << str;
+        return POS_X;
+      }
+      tmp = dict->Find(runes.begin(), runes.end());
+      if (tmp == NULL || tmp->tag.empty()) {
+        return SpecialRule(runes);
+      } else {
+        return tmp->tag;
+      }
+  }
+
+ private:
+  const char* SpecialRule(const RuneStrArray& unicode) const {
+    size_t m = 0;
+    size_t eng = 0;
+    for (size_t i = 0; i < unicode.size() && eng < unicode.size() / 2; i++) {
+      if (unicode[i].rune < 0x80) {
+        eng ++;
+        if ('0' <= unicode[i].rune && unicode[i].rune <= '9') {
+          m++;
+        }
+      }
+    }
+    // ascii char is not found
+    if (eng == 0) {
+      return POS_X;
+    }
+    // all the ascii is number char
+    if (m == eng) {
+      return POS_M;
+    }
+    // the ascii chars contain english letter
+    return POS_ENG;
+  }
+
+}; // class PosTagger
+
+} // namespace cppjieba
+
+#endif

package/submodules/cppjieba/include/cppjieba/PreFilter.hpp

@@ -0,0 +1,54 @@
+#ifndef CPPJIEBA_PRE_FILTER_H
+#define CPPJIEBA_PRE_FILTER_H
+
+#include "Trie.hpp"
+#include "limonp/Logging.hpp"
+
+namespace cppjieba {
+
+class PreFilter {
+ public:
+  //TODO use WordRange instead of Range
+  struct Range {
+    RuneStrArray::const_iterator begin;
+    RuneStrArray::const_iterator end;
+  }; // struct Range
+
+  PreFilter(const unordered_set<Rune>& symbols, 
+        const string& sentence)
+    : symbols_(symbols) {
+    if (!DecodeUTF8RunesInString(sentence, sentence_)) {
+      XLOG(ERROR) << "UTF-8 decode failed for input sentence"; 
+    }
+    cursor_ = sentence_.begin();
+  }
+  ~PreFilter() {
+  }
+  bool HasNext() const {
+    return cursor_ != sentence_.end();
+  }
+  Range Next() {
+    Range range;
+    range.begin = cursor_;
+    while (cursor_ != sentence_.end()) {
+      if (IsIn(symbols_, cursor_->rune)) {
+        if (range.begin == cursor_) {
+          cursor_ ++;
+        }
+        range.end = cursor_;
+        return range;
+      }
+      cursor_ ++;
+    }
+    range.end = sentence_.end();
+    return range;
+  }
+ private:
+  RuneStrArray::const_iterator cursor_;
+  RuneStrArray sentence_;
+  const unordered_set<Rune>& symbols_;
+}; // class PreFilter
+
+} // namespace cppjieba
+
+#endif // CPPJIEBA_PRE_FILTER_H

package/submodules/cppjieba/include/cppjieba/QuerySegment.hpp

@@ -0,0 +1,89 @@
+#ifndef CPPJIEBA_QUERYSEGMENT_H
+#define CPPJIEBA_QUERYSEGMENT_H
+
+#include <algorithm>
+#include <set>
+#include <cassert>
+#include "limonp/Logging.hpp"
+#include "DictTrie.hpp"
+#include "SegmentBase.hpp"
+#include "FullSegment.hpp"
+#include "MixSegment.hpp"
+#include "Unicode.hpp"
+
+namespace cppjieba {
+class QuerySegment: public SegmentBase {
+ public:
+  QuerySegment(const string& dict, const string& model, const string& userDict = "")
+    : mixSeg_(dict, model, userDict),
+      trie_(mixSeg_.GetDictTrie()) {
+  }
+  QuerySegment(const DictTrie* dictTrie, const HMMModel* model)
+    : mixSeg_(dictTrie, model), trie_(dictTrie) {
+  }
+  ~QuerySegment() {
+  }
+
+  void Cut(const string& sentence, vector<string>& words) const {
+    Cut(sentence, words, true);
+  }
+  void Cut(const string& sentence, vector<string>& words, bool hmm) const {
+    vector<Word> tmp;
+    Cut(sentence, tmp, hmm);
+    GetStringsFromWords(tmp, words);
+  }
+  void Cut(const string& sentence, vector<Word>& words, bool hmm = true) const {
+    PreFilter pre_filter(symbols_, sentence);
+    PreFilter::Range range;
+    vector<WordRange> wrs;
+    wrs.reserve(sentence.size()/2);
+    while (pre_filter.HasNext()) {
+      range = pre_filter.Next();
+      Cut(range.begin, range.end, wrs, hmm);
+    }
+    words.clear();
+    words.reserve(wrs.size());
+    GetWordsFromWordRanges(sentence, wrs, words);
+  }
+  void Cut(RuneStrArray::const_iterator begin, RuneStrArray::const_iterator end, vector<WordRange>& res, bool hmm) const {
+    //use mix Cut first
+    vector<WordRange> mixRes;
+    mixSeg_.Cut(begin, end, mixRes, hmm);
+
+    vector<WordRange> fullRes;
+    for (vector<WordRange>::const_iterator mixResItr = mixRes.begin(); mixResItr != mixRes.end(); mixResItr++) {
+      if (mixResItr->Length() > 2) {
+        for (size_t i = 0; i + 1 < mixResItr->Length(); i++) {
+          WordRange wr(mixResItr->left + i, mixResItr->left + i + 1);
+          if (trie_->Find(wr.left, wr.right + 1) != NULL) {
+            res.push_back(wr);
+          }
+        }
+      }
+      if (mixResItr->Length() > 3) {
+        for (size_t i = 0; i + 2 < mixResItr->Length(); i++) {
+          WordRange wr(mixResItr->left + i, mixResItr->left + i + 2);
+          if (trie_->Find(wr.left, wr.right + 1) != NULL) {
+            res.push_back(wr);
+          }
+        }
+      }
+      res.push_back(*mixResItr);
+    }
+  }
+ private:
+  bool IsAllAscii(const Unicode& s) const {
+   for(size_t i = 0; i < s.size(); i++) {
+     if (s[i] >= 0x80) {
+       return false;
+     }
+   }
+   return true;
+  }
+  MixSegment mixSeg_;
+  const DictTrie* trie_;
+}; // QuerySegment
+
+} // namespace cppjieba
+
+#endif

package/submodules/cppjieba/include/cppjieba/SegmentBase.hpp

@@ -0,0 +1,48 @@
+#ifndef CPPJIEBA_SEGMENTBASE_H
+#define CPPJIEBA_SEGMENTBASE_H
+
+#include "limonp/Logging.hpp"
+#include "PreFilter.hpp"
+#include <cassert>
+
+
+namespace cppjieba {
+
+// 修改分隔符，移除空格，只保留其他分隔符
+// 这样英文单词之间的空格不会被当作分隔符
+const char* const SPECIAL_SEPARATORS = "\t\n\xEF\xBC\x8C\xE3\x80\x82";
+
+using namespace limonp;
+
+class SegmentBase {
+ public:
+  SegmentBase() {
+    XCHECK(ResetSeparators(SPECIAL_SEPARATORS));
+  }
+  virtual ~SegmentBase() {
+  }
+
+  virtual void Cut(const string& sentence, vector<string>& words) const = 0;
+
+  bool ResetSeparators(const string& s) {
+    symbols_.clear();
+    RuneStrArray runes;
+    if (!DecodeUTF8RunesInString(s, runes)) {
+      XLOG(ERROR) << "UTF-8 decode failed for separators: " << s;
+      return false;
+    }
+    for (size_t i = 0; i < runes.size(); i++) {
+      if (!symbols_.insert(runes[i].rune).second) {
+        XLOG(ERROR) << s.substr(runes[i].offset, runes[i].len) << " already exists";
+        return false;
+      }
+    }
+    return true;
+  }
+ protected:
+  unordered_set<Rune> symbols_;
+}; // class SegmentBase
+
+} // cppjieba
+
+#endif

package/submodules/cppjieba/include/cppjieba/SegmentTagged.hpp

@@ -0,0 +1,23 @@
+#ifndef CPPJIEBA_SEGMENTTAGGED_H
+#define CPPJIEBA_SEGMENTTAGGED_H
+
+#include "SegmentBase.hpp"
+
+namespace cppjieba {
+
+class SegmentTagged : public SegmentBase{
+ public:
+  SegmentTagged() {
+  }
+  virtual ~SegmentTagged() {
+  }
+
+  virtual bool Tag(const string& src, vector<pair<string, string> >& res) const = 0;
+
+  virtual const DictTrie* GetDictTrie() const = 0;
+
+}; // class SegmentTagged
+
+} // cppjieba
+
+#endif

package/submodules/cppjieba/include/cppjieba/TextRankExtractor.hpp

@@ -0,0 +1,190 @@
+#ifndef CPPJIEBA_TEXTRANK_EXTRACTOR_H
+#define CPPJIEBA_TEXTRANK_EXTRACTOR_H
+
+#include <cmath>
+#include "Jieba.hpp"
+
+namespace cppjieba {
+  using namespace limonp;
+  using namespace std;
+
+  class TextRankExtractor {
+  public:
+    typedef struct _Word {string word;vector<size_t> offsets;double weight;}    Word; // struct Word
+  private:
+    typedef std::map<string,Word> WordMap;
+  
+    class WordGraph{
+    private:
+      typedef double Score;
+      typedef string Node;
+      typedef std::set<Node> NodeSet;
+
+      typedef std::map<Node,double> Edges;
+      typedef std::map<Node,Edges> Graph;
+      //typedef std::unordered_map<Node,double> Edges;
+      //typedef std::unordered_map<Node,Edges> Graph;
+
+      double d;
+      Graph graph;
+      NodeSet nodeSet;
+    public:
+      WordGraph(): d(0.85) {};
+      WordGraph(double in_d): d(in_d) {};
+
+      void addEdge(Node start,Node end,double weight){
+        Edges temp;
+        Edges::iterator gotEdges;
+        nodeSet.insert(start);
+        nodeSet.insert(end);
+        graph[start][end]+=weight;
+        graph[end][start]+=weight;
+      }
+
+      void rank(WordMap &ws,size_t rankTime=10){
+        WordMap outSum;
+        Score wsdef, min_rank, max_rank;
+
+        if( graph.size() == 0)
+          return;
+
+        wsdef = 1.0 / graph.size();
+
+        for(Graph::iterator edges=graph.begin();edges!=graph.end();++edges){
+          // edges->first start节点；edge->first end节点；edge->second 权重
+          ws[edges->first].word=edges->first;
+          ws[edges->first].weight=wsdef;
+          outSum[edges->first].weight=0;
+          for(Edges::iterator edge=edges->second.begin();edge!=edges->second.end();++edge){
+            outSum[edges->first].weight+=edge->second;
+          }
+        }
+        //sort(nodeSet.begin(),nodeSet.end()); 是否需要排序?
+        for( size_t i=0; i<rankTime; i++ ){
+          for(NodeSet::iterator node = nodeSet.begin(); node != nodeSet.end(); node++ ){
+            double s = 0;
+            for( Edges::iterator edge= graph[*node].begin(); edge != graph[*node].end(); edge++ )
+              // edge->first end节点；edge->second 权重
+              s += edge->second / outSum[edge->first].weight * ws[edge->first].weight;
+            ws[*node].weight = (1 - d) + d * s;
+          }
+        }
+
+        min_rank=max_rank=ws.begin()->second.weight;
+        for(WordMap::iterator i = ws.begin(); i != ws.end(); i ++){
+          if( i->second.weight < min_rank ){
+            min_rank = i->second.weight;
+          }
+          if( i->second.weight > max_rank ){
+            max_rank = i->second.weight;
+          }
+        }
+        for(WordMap::iterator i = ws.begin(); i != ws.end(); i ++){
+          ws[i->first].weight = (i->second.weight - min_rank / 10.0) / (max_rank - min_rank / 10.0);
+        }
+      }
+    };
+
+  public: 
+  TextRankExtractor(const string& dictPath, 
+        const string& hmmFilePath, 
+        const string& stopWordPath, 
+        const string& userDict = "") 
+    : segment_(dictPath, hmmFilePath, userDict) {
+    LoadStopWordDict(stopWordPath);
+  }
+  TextRankExtractor(const DictTrie* dictTrie, 
+        const HMMModel* model,
+        const string& stopWordPath) 
+    : segment_(dictTrie, model) {
+    LoadStopWordDict(stopWordPath);
+  }
+    TextRankExtractor(const Jieba& jieba, const string& stopWordPath) : segment_(jieba.GetDictTrie(), jieba.GetHMMModel()) {
+        LoadStopWordDict(stopWordPath);
+    }
+    ~TextRankExtractor() {
+    }
+
+    void Extract(const string& sentence, vector<string>& keywords, size_t topN) const {
+      vector<Word> topWords;
+      Extract(sentence, topWords, topN);
+      for (size_t i = 0; i < topWords.size(); i++) {
+        keywords.push_back(topWords[i].word);
+      }
+    }
+
+    void Extract(const string& sentence, vector<pair<string, double> >& keywords, size_t topN) const {
+      vector<Word> topWords;
+      Extract(sentence, topWords, topN);
+      for (size_t i = 0; i < topWords.size(); i++) {
+        keywords.push_back(pair<string, double>(topWords[i].word, topWords[i].weight));
+      }
+    }
+
+    void Extract(const string& sentence, vector<Word>& keywords, size_t topN, size_t span=5,size_t rankTime=10) const {
+      vector<string> words;
+      segment_.Cut(sentence, words);
+
+      TextRankExtractor::WordGraph graph;
+      WordMap wordmap;
+      size_t offset = 0;
+
+      for(size_t i=0; i < words.size(); i++){
+        size_t t = offset;
+        offset += words[i].size();
+        if (IsSingleWord(words[i]) || stopWords_.find(words[i]) != stopWords_.end()) {
+          continue;
+        }
+        for(size_t j=i+1,skip=0;j<i+span+skip && j<words.size();j++){
+          if (IsSingleWord(words[j]) || stopWords_.find(words[j]) != stopWords_.end()) {
+            skip++;
+            continue;
+          }
+          graph.addEdge(words[i],words[j],1);
+        }
+        wordmap[words[i]].offsets.push_back(t);
+      }
+      if (offset != sentence.size()) {
+        XLOG(ERROR) << "words illegal";
+        return;
+      }
+
+      graph.rank(wordmap,rankTime);
+      
+      keywords.clear();
+      keywords.reserve(wordmap.size());
+      for (WordMap::iterator itr = wordmap.begin(); itr != wordmap.end(); ++itr) {
+        keywords.push_back(itr->second);
+      }
+      
+      topN = min(topN, keywords.size());
+      partial_sort(keywords.begin(), keywords.begin() + topN, keywords.end(), Compare);
+      keywords.resize(topN);
+    }
+  private:
+    void LoadStopWordDict(const string& filePath) {
+      ifstream ifs(filePath.c_str());
+      XCHECK(ifs.is_open()) << "open " << filePath << " failed";
+      string line ;
+      while (getline(ifs, line)) {
+        stopWords_.insert(line);
+      }
+      assert(stopWords_.size());
+    }
+
+    static bool Compare(const Word &x,const Word &y){
+      return x.weight > y.weight;
+    }
+
+    MixSegment segment_;
+    unordered_set<string> stopWords_;
+  }; // class TextRankExtractor
+  
+  inline ostream& operator << (ostream& os, const TextRankExtractor::Word& word) {
+    return os << "{\"word\": \"" << word.word << "\", \"offset\": " << word.offsets << ", \"weight\": " << word.weight << "}"; 
+  }
+} // namespace cppjieba
+
+#endif
+
+