jieba-rb 5.0.0

data/ext/cppjieba/include/cppjieba/SegmentBase.hpp

@@ -0,0 +1,46 @@
+#ifndef CPPJIEBA_SEGMENTBASE_H
+#define CPPJIEBA_SEGMENTBASE_H
+
+#include "limonp/Logging.hpp"
+#include "PreFilter.hpp"
+#include <cassert>
+
+
+namespace cppjieba {
+
+const char* const SPECIAL_SEPARATORS = " \t\n\xEF\xBC\x8C\xE3\x80\x82";
+
+using namespace limonp;
+
+class SegmentBase {
+ public:
+  SegmentBase() {
+    XCHECK(ResetSeparators(SPECIAL_SEPARATORS));
+  }
+  virtual ~SegmentBase() {
+  }
+
+  virtual void Cut(const string& sentence, vector<string>& words) const = 0;
+
+  bool ResetSeparators(const string& s) {
+    symbols_.clear();
+    RuneStrArray runes;
+    if (!DecodeRunesInString(s, runes)) {
+      XLOG(ERROR) << "decode " << s << " failed";
+      return false;
+    }
+    for (size_t i = 0; i < runes.size(); i++) {
+      if (!symbols_.insert(runes[i].rune).second) {
+        XLOG(ERROR) << s.substr(runes[i].offset, runes[i].len) << " already exists";
+        return false;
+      }
+    }
+    return true;
+  }
+ protected:
+  unordered_set<Rune> symbols_;
+}; // class SegmentBase
+
+} // cppjieba
+
+#endif

data/ext/cppjieba/include/cppjieba/SegmentTagged.hpp

@@ -0,0 +1,24 @@
+#ifndef CPPJIEBA_SEGMENTTAGGED_H
+#define CPPJIEBA_SEGMENTTAGGED_H
+
+#include "SegmentBase.hpp"
+#include "DictTrie.hpp"
+
+namespace cppjieba {
+
+class SegmentTagged : public SegmentBase{
+ public:
+  SegmentTagged() {
+  }
+  virtual ~SegmentTagged() {
+  }
+
+  virtual bool Tag(const string& src, vector<pair<string, string> >& res) const = 0;
+
+  virtual const DictTrie* GetDictTrie() const = 0;
+
+}; // class SegmentTagged
+
+} // cppjieba
+
+#endif

data/ext/cppjieba/include/cppjieba/TextRankExtractor.hpp

@@ -0,0 +1,190 @@
+#ifndef CPPJIEBA_TEXTRANK_EXTRACTOR_H
+#define CPPJIEBA_TEXTRANK_EXTRACTOR_H
+
+#include <cmath>
+#include "Jieba.hpp"
+
+namespace cppjieba {
+  using namespace limonp;
+  using namespace std;
+
+  class TextRankExtractor {
+  public:
+    typedef struct _Word {string word;vector<size_t> offsets;double weight;}    Word; // struct Word
+  private:
+    typedef std::map<string,Word> WordMap;
+  
+    class WordGraph{
+    private:
+      typedef double Score;
+      typedef string Node;
+      typedef std::set<Node> NodeSet;
+
+      typedef std::map<Node,double> Edges;
+      typedef std::map<Node,Edges> Graph;
+      //typedef std::unordered_map<Node,double> Edges;
+      //typedef std::unordered_map<Node,Edges> Graph;
+
+      double d;
+      Graph graph;
+      NodeSet nodeSet;
+    public:
+      WordGraph(): d(0.85) {};
+      WordGraph(double in_d): d(in_d) {};
+
+      void addEdge(Node start,Node end,double weight){
+        Edges temp;
+        Edges::iterator gotEdges;
+        nodeSet.insert(start);
+        nodeSet.insert(end);
+        graph[start][end]+=weight;
+        graph[end][start]+=weight;
+      }
+
+      void rank(WordMap &ws,size_t rankTime=10){
+        WordMap outSum;
+        Score wsdef, min_rank, max_rank;
+
+        if( graph.size() == 0)
+          return;
+
+        wsdef = 1.0 / graph.size();
+
+        for(Graph::iterator edges=graph.begin();edges!=graph.end();++edges){
+          // edges->first start节点；edge->first end节点；edge->second 权重
+          ws[edges->first].word=edges->first;
+          ws[edges->first].weight=wsdef;
+          outSum[edges->first].weight=0;
+          for(Edges::iterator edge=edges->second.begin();edge!=edges->second.end();++edge){
+            outSum[edges->first].weight+=edge->second;
+          }
+        }
+        //sort(nodeSet.begin(),nodeSet.end()); 是否需要排序?
+        for( size_t i=0; i<rankTime; i++ ){
+          for(NodeSet::iterator node = nodeSet.begin(); node != nodeSet.end(); node++ ){
+            double s = 0;
+            for( Edges::iterator edge= graph[*node].begin(); edge != graph[*node].end(); edge++ )
+              // edge->first end节点；edge->second 权重
+              s += edge->second / outSum[edge->first].weight * ws[edge->first].weight;
+            ws[*node].weight = (1 - d) + d * s;
+          }
+        }
+
+        min_rank=max_rank=ws.begin()->second.weight;
+        for(WordMap::iterator i = ws.begin(); i != ws.end(); i ++){
+          if( i->second.weight < min_rank ){
+            min_rank = i->second.weight;
+          }
+          if( i->second.weight > max_rank ){
+            max_rank = i->second.weight;
+          }
+        }
+        for(WordMap::iterator i = ws.begin(); i != ws.end(); i ++){
+          ws[i->first].weight = (i->second.weight - min_rank / 10.0) / (max_rank - min_rank / 10.0);
+        }
+      }
+    };
+
+  public: 
+  TextRankExtractor(const string& dictPath, 
+        const string& hmmFilePath, 
+        const string& stopWordPath, 
+        const string& userDict = "") 
+    : segment_(dictPath, hmmFilePath, userDict) {
+    LoadStopWordDict(stopWordPath);
+  }
+  TextRankExtractor(const DictTrie* dictTrie, 
+        const HMMModel* model,
+        const string& stopWordPath) 
+    : segment_(dictTrie, model) {
+    LoadStopWordDict(stopWordPath);
+  }
+    TextRankExtractor(const Jieba& jieba, const string& stopWordPath) : segment_(jieba.GetDictTrie(), jieba.GetHMMModel()) {
+        LoadStopWordDict(stopWordPath);
+    }
+    ~TextRankExtractor() {
+    }
+
+    void Extract(const string& sentence, vector<string>& keywords, size_t topN) const {
+      vector<Word> topWords;
+      Extract(sentence, topWords, topN);
+      for (size_t i = 0; i < topWords.size(); i++) {
+        keywords.push_back(topWords[i].word);
+      }
+    }
+
+    void Extract(const string& sentence, vector<pair<string, double> >& keywords, size_t topN) const {
+      vector<Word> topWords;
+      Extract(sentence, topWords, topN);
+      for (size_t i = 0; i < topWords.size(); i++) {
+        keywords.push_back(pair<string, double>(topWords[i].word, topWords[i].weight));
+      }
+    }
+
+    void Extract(const string& sentence, vector<Word>& keywords, size_t topN, size_t span=5,size_t rankTime=10) const {
+      vector<string> words;
+      segment_.Cut(sentence, words);
+
+      TextRankExtractor::WordGraph graph;
+      WordMap wordmap;
+      size_t offset = 0;
+
+      for(size_t i=0; i < words.size(); i++){
+        size_t t = offset;
+        offset += words[i].size();
+        if (IsSingleWord(words[i]) || stopWords_.find(words[i]) != stopWords_.end()) {
+          continue;
+        }
+        for(size_t j=i+1,skip=0;j<i+span+skip && j<words.size();j++){
+          if (IsSingleWord(words[j]) || stopWords_.find(words[j]) != stopWords_.end()) {
+            skip++;
+            continue;
+          }
+          graph.addEdge(words[i],words[j],1);
+        }
+        wordmap[words[i]].offsets.push_back(t);
+      }
+      if (offset != sentence.size()) {
+        XLOG(ERROR) << "words illegal";
+        return;
+      }
+
+      graph.rank(wordmap,rankTime);
+      
+      keywords.clear();
+      keywords.reserve(wordmap.size());
+      for (WordMap::iterator itr = wordmap.begin(); itr != wordmap.end(); ++itr) {
+        keywords.push_back(itr->second);
+      }
+      
+      topN = min(topN, keywords.size());
+      partial_sort(keywords.begin(), keywords.begin() + topN, keywords.end(), Compare);
+      keywords.resize(topN);
+    }
+  private:
+    void LoadStopWordDict(const string& filePath) {
+      ifstream ifs(filePath.c_str());
+      XCHECK(ifs.is_open()) << "open " << filePath << " failed";
+      string line ;
+      while (getline(ifs, line)) {
+        stopWords_.insert(line);
+      }
+      assert(stopWords_.size());
+    }
+
+    static bool Compare(const Word &x,const Word &y){
+      return x.weight > y.weight;
+    }
+
+    MixSegment segment_;
+    unordered_set<string> stopWords_;
+  }; // class TextRankExtractor
+  
+  inline ostream& operator << (ostream& os, const TextRankExtractor::Word& word) {
+    return os << "{\"word\": \"" << word.word << "\", \"offset\": " << word.offsets << ", \"weight\": " << word.weight << "}"; 
+  }
+} // namespace cppjieba
+
+#endif
+
+

data/ext/cppjieba/include/cppjieba/Trie.hpp

@@ -0,0 +1,174 @@
+#ifndef CPPJIEBA_TRIE_HPP
+#define CPPJIEBA_TRIE_HPP
+
+#include <vector>
+#include <queue>
+#include "limonp/StdExtension.hpp"
+#include "Unicode.hpp"
+
+namespace cppjieba {
+
+using namespace std;
+
+const size_t MAX_WORD_LENGTH = 512;
+
+struct DictUnit {
+  Unicode word;
+  double weight;
+  string tag;
+}; // struct DictUnit
+
+// for debugging
+// inline ostream & operator << (ostream& os, const DictUnit& unit) {
+//   string s;
+//   s << unit.word;
+//   return os << StringFormat("%s %s %.3lf", s.c_str(), unit.tag.c_str(), unit.weight);
+// }
+
+struct Dag {
+  RuneStr runestr;
+  // [offset, nexts.first]
+  limonp::LocalVector<pair<size_t, const DictUnit*> > nexts;
+  const DictUnit * pInfo;
+  double weight;
+  size_t nextPos; // TODO
+  Dag():runestr(), pInfo(NULL), weight(0.0), nextPos(0) {
+  }
+}; // struct Dag
+
+typedef Rune TrieKey;
+
+class TrieNode {
+ public :
+  TrieNode(): next(NULL), ptValue(NULL) {
+  }
+ public:
+  typedef unordered_map<TrieKey, TrieNode*> NextMap;
+  NextMap *next;
+  const DictUnit *ptValue;
+};
+
+class Trie {
+ public:
+  Trie(const vector<Unicode>& keys, const vector<const DictUnit*>& valuePointers)
+   : root_(new TrieNode) {
+    CreateTrie(keys, valuePointers);
+  }
+  ~Trie() {
+    DeleteNode(root_);
+  }
+
+  const DictUnit* Find(RuneStrArray::const_iterator begin, RuneStrArray::const_iterator end) const {
+    if (begin == end) {
+      return NULL;
+    }
+
+    const TrieNode* ptNode = root_;
+    TrieNode::NextMap::const_iterator citer;
+    for (RuneStrArray::const_iterator it = begin; it != end; it++) {
+      if (NULL == ptNode->next) {
+        return NULL;
+      }
+      citer = ptNode->next->find(it->rune);
+      if (ptNode->next->end() == citer) {
+        return NULL;
+      }
+      ptNode = citer->second;
+    }
+    return ptNode->ptValue;
+  }
+
+  void Find(RuneStrArray::const_iterator begin, 
+        RuneStrArray::const_iterator end, 
+        vector<struct Dag>&res, 
+        size_t max_word_len = MAX_WORD_LENGTH) const {
+    assert(root_ != NULL);
+    res.resize(end - begin);
+
+    const TrieNode *ptNode = NULL;
+    TrieNode::NextMap::const_iterator citer;
+    for (size_t i = 0; i < size_t(end - begin); i++) {
+      res[i].runestr = *(begin + i);
+
+      if (root_->next != NULL && root_->next->end() != (citer = root_->next->find(res[i].runestr.rune))) {
+        ptNode = citer->second;
+      } else {
+        ptNode = NULL;
+      }
+      if (ptNode != NULL) {
+        res[i].nexts.push_back(pair<size_t, const DictUnit*>(i, ptNode->ptValue));
+      } else {
+        res[i].nexts.push_back(pair<size_t, const DictUnit*>(i, static_cast<const DictUnit*>(NULL)));
+      }
+
+      for (size_t j = i + 1; j < size_t(end - begin) && (j - i + 1) <= max_word_len; j++) {
+        if (ptNode == NULL || ptNode->next == NULL) {
+          break;
+        }
+        citer = ptNode->next->find((begin + j)->rune);
+        if (ptNode->next->end() == citer) {
+          break;
+        }
+        ptNode = citer->second;
+        if (NULL != ptNode->ptValue) {
+          res[i].nexts.push_back(pair<size_t, const DictUnit*>(j, ptNode->ptValue));
+        }
+      }
+    }
+  }
+
+  void InsertNode(const Unicode& key, const DictUnit* ptValue) {
+    if (key.begin() == key.end()) {
+      return;
+    }
+
+    TrieNode::NextMap::const_iterator kmIter;
+    TrieNode *ptNode = root_;
+    for (Unicode::const_iterator citer = key.begin(); citer != key.end(); ++citer) {
+      if (NULL == ptNode->next) {
+        ptNode->next = new TrieNode::NextMap;
+      }
+      kmIter = ptNode->next->find(*citer);
+      if (ptNode->next->end() == kmIter) {
+        TrieNode *nextNode = new TrieNode;
+
+        ptNode->next->insert(make_pair(*citer, nextNode));
+        ptNode = nextNode;
+      } else {
+        ptNode = kmIter->second;
+      }
+    }
+    assert(ptNode != NULL);
+    ptNode->ptValue = ptValue;
+  }
+
+ private:
+  void CreateTrie(const vector<Unicode>& keys, const vector<const DictUnit*>& valuePointers) {
+    if (valuePointers.empty() || keys.empty()) {
+      return;
+    }
+    assert(keys.size() == valuePointers.size());
+
+    for (size_t i = 0; i < keys.size(); i++) {
+      InsertNode(keys[i], valuePointers[i]);
+    }
+  }
+
+  void DeleteNode(TrieNode* node) {
+    if (NULL == node) {
+      return;
+    }
+    if (NULL != node->next) {
+      for (TrieNode::NextMap::iterator it = node->next->begin(); it != node->next->end(); ++it) {
+        DeleteNode(it->second);
+      }
+      delete node->next;
+    }
+    delete node;
+  }
+
+  TrieNode* root_;
+}; // class Trie
+} // namespace cppjieba
+
+#endif // CPPJIEBA_TRIE_HPP

data/ext/cppjieba/include/cppjieba/Unicode.hpp

@@ -0,0 +1,215 @@
+#ifndef CPPJIEBA_UNICODE_H
+#define CPPJIEBA_UNICODE_H
+
+#include <stdint.h>
+#include <stdlib.h>
+#include <string>
+#include <vector>
+#include <ostream>
+#include "limonp/LocalVector.hpp"
+
+namespace cppjieba {
+
+using std::string;
+using std::vector;
+
+typedef uint32_t Rune;
+
+struct Word {
+  string word;
+  uint32_t offset;
+  Word(const string& w, uint32_t o)
+   : word(w), offset(o) {
+  }
+}; // struct Word
+
+inline std::ostream& operator << (std::ostream& os, const Word& w) {
+  return os << "{\"word\": \"" << w.word << "\", \"offset\": " << w.offset << "}";
+}
+
+struct RuneStr {
+  Rune rune;
+  uint32_t offset;
+  uint32_t len;
+  RuneStr(): rune(0), offset(0), len(0) {
+  }
+  RuneStr(Rune r, uint32_t o, uint32_t l)
+    : rune(r), offset(o), len(l) {
+  }
+}; // struct RuneStr
+
+inline std::ostream& operator << (std::ostream& os, const RuneStr& r) {
+  return os << "{\"rune\": \"" << r.rune << "\", \"offset\": " << r.offset << ", \"len\": " << r.len << "}";
+}
+
+typedef limonp::LocalVector<Rune> Unicode;
+typedef limonp::LocalVector<struct RuneStr> RuneStrArray;
+
+// [left, right]
+struct WordRange {
+  RuneStrArray::const_iterator left;
+  RuneStrArray::const_iterator right;
+  WordRange(RuneStrArray::const_iterator l, RuneStrArray::const_iterator r)
+   : left(l), right(r) {
+  }
+  size_t Length() const {
+    return right - left + 1;
+  }
+  bool IsAllAscii() const {
+    for (RuneStrArray::const_iterator iter = left; iter <= right; ++iter) {
+      if (iter->rune >= 0x80) {
+        return false;
+      }
+    }
+    return true;
+  }
+}; // struct WordRange
+
+struct RuneStrLite {
+  uint32_t rune;
+  uint32_t len;
+  RuneStrLite(): rune(0), len(0) {
+  }
+  RuneStrLite(uint32_t r, uint32_t l): rune(r), len(l) {
+  }
+}; // struct RuneStrLite
+
+inline RuneStrLite DecodeRuneInString(const char* str, size_t len) {
+  RuneStrLite rp(0, 0);
+  if (str == NULL || len == 0) {
+    return rp;
+  }
+  if (!(str[0] & 0x80)) { // 0xxxxxxx
+    // 7bit, total 7bit
+    rp.rune = (uint8_t)(str[0]) & 0x7f;
+    rp.len = 1;
+  } else if ((uint8_t)str[0] <= 0xdf &&  1 < len) { 
+    // 110xxxxxx
+    // 5bit, total 5bit
+    rp.rune = (uint8_t)(str[0]) & 0x1f;
+
+    // 6bit, total 11bit
+    rp.rune <<= 6;
+    rp.rune |= (uint8_t)(str[1]) & 0x3f;
+    rp.len = 2;
+  } else if((uint8_t)str[0] <= 0xef && 2 < len) { // 1110xxxxxx
+    // 4bit, total 4bit
+    rp.rune = (uint8_t)(str[0]) & 0x0f;
+
+    // 6bit, total 10bit
+    rp.rune <<= 6;
+    rp.rune |= (uint8_t)(str[1]) & 0x3f;
+
+    // 6bit, total 16bit
+    rp.rune <<= 6;
+    rp.rune |= (uint8_t)(str[2]) & 0x3f;
+
+    rp.len = 3;
+  } else if((uint8_t)str[0] <= 0xf7 && 3 < len) { // 11110xxxx
+    // 3bit, total 3bit
+    rp.rune = (uint8_t)(str[0]) & 0x07;
+
+    // 6bit, total 9bit
+    rp.rune <<= 6;
+    rp.rune |= (uint8_t)(str[1]) & 0x3f;
+
+    // 6bit, total 15bit
+    rp.rune <<= 6;
+    rp.rune |= (uint8_t)(str[2]) & 0x3f;
+
+    // 6bit, total 21bit
+    rp.rune <<= 6;
+    rp.rune |= (uint8_t)(str[3]) & 0x3f;
+
+    rp.len = 4;
+  } else {
+    rp.rune = 0;
+    rp.len = 0;
+  }
+  return rp;
+}
+
+inline bool DecodeRunesInString(const char* s, size_t len, RuneStrArray& runes) {
+  runes.clear();
+  runes.reserve(len / 2);
+  for (size_t i = 0; i < len;) {
+    RuneStrLite rp = DecodeRuneInString(s + i, len - i);
+    if (rp.len == 0) {
+      runes.clear();
+      return false;
+    }
+    RuneStr x(rp.rune, i, rp.len);
+    runes.push_back(x);
+    i += rp.len;
+  }
+  return true;
+}
+
+inline bool DecodeRunesInString(const string& s, RuneStrArray& runes) {
+  return DecodeRunesInString(s.c_str(), s.size(), runes);
+}
+
+inline bool DecodeRunesInString(const char* s, size_t len, Unicode& unicode) {
+  unicode.clear();
+  RuneStrArray runes;
+  if (!DecodeRunesInString(s, len, runes)) {
+    return false;
+  }
+  unicode.reserve(runes.size());
+  for (size_t i = 0; i < runes.size(); i++) {
+    unicode.push_back(runes[i].rune);
+  }
+  return true;
+}
+
+inline bool IsSingleWord(const string& str) {
+  RuneStrLite rp = DecodeRuneInString(str.c_str(), str.size());
+  return rp.len == str.size();
+}
+
+inline bool DecodeRunesInString(const string& s, Unicode& unicode) {
+  return DecodeRunesInString(s.c_str(), s.size(), unicode);
+}
+
+inline Unicode DecodeRunesInString(const string& s) {
+  Unicode result;
+  DecodeRunesInString(s, result);
+  return result;
+}
+
+
+// [left, right]
+inline Word GetWordFromRunes(const string& s, RuneStrArray::const_iterator left, RuneStrArray::const_iterator right) {
+  assert(right->offset >= left->offset);
+  uint32_t len = right->offset - left->offset + right->len;
+  return Word(s.substr(left->offset, len), left->offset);
+}
+
+inline string GetStringFromRunes(const string& s, RuneStrArray::const_iterator left, RuneStrArray::const_iterator right) {
+  assert(right->offset >= left->offset);
+  uint32_t len = right->offset - left->offset + right->len;
+  return s.substr(left->offset, len);
+}
+
+inline void GetWordsFromWordRanges(const string& s, const vector<WordRange>& wrs, vector<Word>& words) {
+  for (size_t i = 0; i < wrs.size(); i++) {
+    words.push_back(GetWordFromRunes(s, wrs[i].left, wrs[i].right));
+  }
+}
+
+inline vector<Word> GetWordsFromWordRanges(const string& s, const vector<WordRange>& wrs) {
+  vector<Word> result;
+  GetWordsFromWordRanges(s, wrs, result);
+  return result;
+}
+
+inline void GetStringsFromWords(const vector<Word>& words, vector<string>& strs) {
+  strs.resize(words.size());
+  for (size_t i = 0; i < words.size(); ++i) {
+    strs[i] = words[i].word;
+  }
+}
+
+} // namespace cppjieba
+
+#endif // CPPJIEBA_UNICODE_H