cppjieba_rb 0.3.3 → 0.4.0

data/ext/cppjieba/include/cppjieba/SegmentBase.hpp

@@ -1,46 +0,0 @@
-#ifndef CPPJIEBA_SEGMENTBASE_H
-#define CPPJIEBA_SEGMENTBASE_H
-
-#include "limonp/Logging.hpp"
-#include "PreFilter.hpp"
-#include <cassert>
-
-
-namespace cppjieba {
-
-const char* const SPECIAL_SEPARATORS = " \t\n\xEF\xBC\x8C\xE3\x80\x82";
-
-using namespace limonp;
-
-class SegmentBase {
- public:
-  SegmentBase() {
-    XCHECK(ResetSeparators(SPECIAL_SEPARATORS));
-  }
-  virtual ~SegmentBase() {
-  }
-
-  virtual void Cut(const string& sentence, vector<string>& words) const = 0;
-
-  bool ResetSeparators(const string& s) {
-    symbols_.clear();
-    RuneStrArray runes;
-    if (!DecodeRunesInString(s, runes)) {
-      XLOG(ERROR) << "decode " << s << " failed";
-      return false;
-    }
-    for (size_t i = 0; i < runes.size(); i++) {
-      if (!symbols_.insert(runes[i].rune).second) {
-        XLOG(ERROR) << s.substr(runes[i].offset, runes[i].len) << " already exists";
-        return false;
-      }
-    }
-    return true;
-  }
- protected:
-  unordered_set<Rune> symbols_;
-}; // class SegmentBase
-
-} // cppjieba
-
-#endif

data/ext/cppjieba/include/cppjieba/SegmentTagged.hpp

@@ -1,23 +0,0 @@
-#ifndef CPPJIEBA_SEGMENTTAGGED_H
-#define CPPJIEBA_SEGMENTTAGGED_H
-
-#include "SegmentBase.hpp"
-
-namespace cppjieba {
-
-class SegmentTagged : public SegmentBase{
- public:
-  SegmentTagged() {
-  }
-  virtual ~SegmentTagged() {
-  }
-
-  virtual bool Tag(const string& src, vector<pair<string, string> >& res) const = 0;
-
-  virtual const DictTrie* GetDictTrie() const = 0;
-
-}; // class SegmentTagged
-
-} // cppjieba
-
-#endif

data/ext/cppjieba/include/cppjieba/TextRankExtractor.hpp

@@ -1,190 +0,0 @@
-#ifndef CPPJIEBA_TEXTRANK_EXTRACTOR_H
-#define CPPJIEBA_TEXTRANK_EXTRACTOR_H
-
-#include <cmath>
-#include "Jieba.hpp"
-
-namespace cppjieba {
-  using namespace limonp;
-  using namespace std;
-
-  class TextRankExtractor {
-  public:
-    typedef struct _Word {string word;vector<size_t> offsets;double weight;}    Word; // struct Word
-  private:
-    typedef std::map<string,Word> WordMap;
-  
-    class WordGraph{
-    private:
-      typedef double Score;
-      typedef string Node;
-      typedef std::set<Node> NodeSet;
-
-      typedef std::map<Node,double> Edges;
-      typedef std::map<Node,Edges> Graph;
-      //typedef std::unordered_map<Node,double> Edges;
-      //typedef std::unordered_map<Node,Edges> Graph;
-
-      double d;
-      Graph graph;
-      NodeSet nodeSet;
-    public:
-      WordGraph(): d(0.85) {};
-      WordGraph(double in_d): d(in_d) {};
-
-      void addEdge(Node start,Node end,double weight){
-        Edges temp;
-        Edges::iterator gotEdges;
-        nodeSet.insert(start);
-        nodeSet.insert(end);
-        graph[start][end]+=weight;
-        graph[end][start]+=weight;
-      }
-
-      void rank(WordMap &ws,size_t rankTime=10){
-        WordMap outSum;
-        Score wsdef, min_rank, max_rank;
-
-        if( graph.size() == 0)
-          return;
-
-        wsdef = 1.0 / graph.size();
-
-        for(Graph::iterator edges=graph.begin();edges!=graph.end();++edges){
-          // edges->first start节点；edge->first end节点；edge->second 权重
-          ws[edges->first].word=edges->first;
-          ws[edges->first].weight=wsdef;
-          outSum[edges->first].weight=0;
-          for(Edges::iterator edge=edges->second.begin();edge!=edges->second.end();++edge){
-            outSum[edges->first].weight+=edge->second;
-          }
-        }
-        //sort(nodeSet.begin(),nodeSet.end()); 是否需要排序?
-        for( size_t i=0; i<rankTime; i++ ){
-          for(NodeSet::iterator node = nodeSet.begin(); node != nodeSet.end(); node++ ){
-            double s = 0;
-            for( Edges::iterator edge= graph[*node].begin(); edge != graph[*node].end(); edge++ )
-              // edge->first end节点；edge->second 权重
-              s += edge->second / outSum[edge->first].weight * ws[edge->first].weight;
-            ws[*node].weight = (1 - d) + d * s;
-          }
-        }
-
-        min_rank=max_rank=ws.begin()->second.weight;
-        for(WordMap::iterator i = ws.begin(); i != ws.end(); i ++){
-          if( i->second.weight < min_rank ){
-            min_rank = i->second.weight;
-          }
-          if( i->second.weight > max_rank ){
-            max_rank = i->second.weight;
-          }
-        }
-        for(WordMap::iterator i = ws.begin(); i != ws.end(); i ++){
-          ws[i->first].weight = (i->second.weight - min_rank / 10.0) / (max_rank - min_rank / 10.0);
-        }
-      }
-    };
-
-  public: 
-  TextRankExtractor(const string& dictPath, 
-        const string& hmmFilePath, 
-        const string& stopWordPath, 
-        const string& userDict = "") 
-    : segment_(dictPath, hmmFilePath, userDict) {
-    LoadStopWordDict(stopWordPath);
-  }
-  TextRankExtractor(const DictTrie* dictTrie, 
-        const HMMModel* model,
-        const string& stopWordPath) 
-    : segment_(dictTrie, model) {
-    LoadStopWordDict(stopWordPath);
-  }
-    TextRankExtractor(const Jieba& jieba, const string& stopWordPath) : segment_(jieba.GetDictTrie(), jieba.GetHMMModel()) {
-        LoadStopWordDict(stopWordPath);
-    }
-    ~TextRankExtractor() {
-    }
-
-    void Extract(const string& sentence, vector<string>& keywords, size_t topN) const {
-      vector<Word> topWords;
-      Extract(sentence, topWords, topN);
-      for (size_t i = 0; i < topWords.size(); i++) {
-        keywords.push_back(topWords[i].word);
-      }
-    }
-
-    void Extract(const string& sentence, vector<pair<string, double> >& keywords, size_t topN) const {
-      vector<Word> topWords;
-      Extract(sentence, topWords, topN);
-      for (size_t i = 0; i < topWords.size(); i++) {
-        keywords.push_back(pair<string, double>(topWords[i].word, topWords[i].weight));
-      }
-    }
-
-    void Extract(const string& sentence, vector<Word>& keywords, size_t topN, size_t span=5,size_t rankTime=10) const {
-      vector<string> words;
-      segment_.Cut(sentence, words);
-
-      TextRankExtractor::WordGraph graph;
-      WordMap wordmap;
-      size_t offset = 0;
-
-      for(size_t i=0; i < words.size(); i++){
-        size_t t = offset;
-        offset += words[i].size();
-        if (IsSingleWord(words[i]) || stopWords_.find(words[i]) != stopWords_.end()) {
-          continue;
-        }
-        for(size_t j=i+1,skip=0;j<i+span+skip && j<words.size();j++){
-          if (IsSingleWord(words[j]) || stopWords_.find(words[j]) != stopWords_.end()) {
-            skip++;
-            continue;
-          }
-          graph.addEdge(words[i],words[j],1);
-        }
-        wordmap[words[i]].offsets.push_back(t);
-      }
-      if (offset != sentence.size()) {
-        XLOG(ERROR) << "words illegal";
-        return;
-      }
-
-      graph.rank(wordmap,rankTime);
-      
-      keywords.clear();
-      keywords.reserve(wordmap.size());
-      for (WordMap::iterator itr = wordmap.begin(); itr != wordmap.end(); ++itr) {
-        keywords.push_back(itr->second);
-      }
-      
-      topN = min(topN, keywords.size());
-      partial_sort(keywords.begin(), keywords.begin() + topN, keywords.end(), Compare);
-      keywords.resize(topN);
-    }
-  private:
-    void LoadStopWordDict(const string& filePath) {
-      ifstream ifs(filePath.c_str());
-      XCHECK(ifs.is_open()) << "open " << filePath << " failed";
-      string line ;
-      while (getline(ifs, line)) {
-        stopWords_.insert(line);
-      }
-      assert(stopWords_.size());
-    }
-
-    static bool Compare(const Word &x,const Word &y){
-      return x.weight > y.weight;
-    }
-
-    MixSegment segment_;
-    unordered_set<string> stopWords_;
-  }; // class TextRankExtractor
-  
-  inline ostream& operator << (ostream& os, const TextRankExtractor::Word& word) {
-    return os << "{\"word\": \"" << word.word << "\", \"offset\": " << word.offsets << ", \"weight\": " << word.weight << "}"; 
-  }
-} // namespace cppjieba
-
-#endif
-
-

data/ext/cppjieba/include/cppjieba/Trie.hpp

@@ -1,174 +0,0 @@
-#ifndef CPPJIEBA_TRIE_HPP
-#define CPPJIEBA_TRIE_HPP
-
-#include <vector>
-#include <queue>
-#include "limonp/StdExtension.hpp"
-#include "Unicode.hpp"
-
-namespace cppjieba {
-
-using namespace std;
-
-const size_t MAX_WORD_LENGTH = 512;
-
-struct DictUnit {
-  Unicode word;
-  double weight;
-  string tag;
-}; // struct DictUnit
-
-// for debugging
-// inline ostream & operator << (ostream& os, const DictUnit& unit) {
-//   string s;
-//   s << unit.word;
-//   return os << StringFormat("%s %s %.3lf", s.c_str(), unit.tag.c_str(), unit.weight);
-// }
-
-struct Dag {
-  RuneStr runestr;
-  // [offset, nexts.first]
-  limonp::LocalVector<pair<size_t, const DictUnit*> > nexts;
-  const DictUnit * pInfo;
-  double weight;
-  size_t nextPos; // TODO
-  Dag():runestr(), pInfo(NULL), weight(0.0), nextPos(0) {
-  }
-}; // struct Dag
-
-typedef Rune TrieKey;
-
-class TrieNode {
- public :
-  TrieNode(): next(NULL), ptValue(NULL) {
-  }
- public:
-  typedef unordered_map<TrieKey, TrieNode*> NextMap;
-  NextMap *next;
-  const DictUnit *ptValue;
-};
-
-class Trie {
- public:
-  Trie(const vector<Unicode>& keys, const vector<const DictUnit*>& valuePointers)
-   : root_(new TrieNode) {
-    CreateTrie(keys, valuePointers);
-  }
-  ~Trie() {
-    DeleteNode(root_);
-  }
-
-  const DictUnit* Find(RuneStrArray::const_iterator begin, RuneStrArray::const_iterator end) const {
-    if (begin == end) {
-      return NULL;
-    }
-
-    const TrieNode* ptNode = root_;
-    TrieNode::NextMap::const_iterator citer;
-    for (RuneStrArray::const_iterator it = begin; it != end; it++) {
-      if (NULL == ptNode->next) {
-        return NULL;
-      }
-      citer = ptNode->next->find(it->rune);
-      if (ptNode->next->end() == citer) {
-        return NULL;
-      }
-      ptNode = citer->second;
-    }
-    return ptNode->ptValue;
-  }
-
-  void Find(RuneStrArray::const_iterator begin, 
-        RuneStrArray::const_iterator end, 
-        vector<struct Dag>&res, 
-        size_t max_word_len = MAX_WORD_LENGTH) const {
-    assert(root_ != NULL);
-    res.resize(end - begin);
-
-    const TrieNode *ptNode = NULL;
-    TrieNode::NextMap::const_iterator citer;
-    for (size_t i = 0; i < size_t(end - begin); i++) {
-      res[i].runestr = *(begin + i);
-
-      if (root_->next != NULL && root_->next->end() != (citer = root_->next->find(res[i].runestr.rune))) {
-        ptNode = citer->second;
-      } else {
-        ptNode = NULL;
-      }
-      if (ptNode != NULL) {
-        res[i].nexts.push_back(pair<size_t, const DictUnit*>(i, ptNode->ptValue));
-      } else {
-        res[i].nexts.push_back(pair<size_t, const DictUnit*>(i, static_cast<const DictUnit*>(NULL)));
-      }
-
-      for (size_t j = i + 1; j < size_t(end - begin) && (j - i + 1) <= max_word_len; j++) {
-        if (ptNode == NULL || ptNode->next == NULL) {
-          break;
-        }
-        citer = ptNode->next->find((begin + j)->rune);
-        if (ptNode->next->end() == citer) {
-          break;
-        }
-        ptNode = citer->second;
-        if (NULL != ptNode->ptValue) {
-          res[i].nexts.push_back(pair<size_t, const DictUnit*>(j, ptNode->ptValue));
-        }
-      }
-    }
-  }
-
-  void InsertNode(const Unicode& key, const DictUnit* ptValue) {
-    if (key.begin() == key.end()) {
-      return;
-    }
-
-    TrieNode::NextMap::const_iterator kmIter;
-    TrieNode *ptNode = root_;
-    for (Unicode::const_iterator citer = key.begin(); citer != key.end(); ++citer) {
-      if (NULL == ptNode->next) {
-        ptNode->next = new TrieNode::NextMap;
-      }
-      kmIter = ptNode->next->find(*citer);
-      if (ptNode->next->end() == kmIter) {
-        TrieNode *nextNode = new TrieNode;
-
-        ptNode->next->insert(make_pair(*citer, nextNode));
-        ptNode = nextNode;
-      } else {
-        ptNode = kmIter->second;
-      }
-    }
-    assert(ptNode != NULL);
-    ptNode->ptValue = ptValue;
-  }
-
- private:
-  void CreateTrie(const vector<Unicode>& keys, const vector<const DictUnit*>& valuePointers) {
-    if (valuePointers.empty() || keys.empty()) {
-      return;
-    }
-    assert(keys.size() == valuePointers.size());
-
-    for (size_t i = 0; i < keys.size(); i++) {
-      InsertNode(keys[i], valuePointers[i]);
-    }
-  }
-
-  void DeleteNode(TrieNode* node) {
-    if (NULL == node) {
-      return;
-    }
-    if (NULL != node->next) {
-      for (TrieNode::NextMap::iterator it = node->next->begin(); it != node->next->end(); ++it) {
-        DeleteNode(it->second);
-      }
-      delete node->next;
-    }
-    delete node;
-  }
-
-  TrieNode* root_;
-}; // class Trie
-} // namespace cppjieba
-
-#endif // CPPJIEBA_TRIE_HPP

data/ext/cppjieba/include/cppjieba/Unicode.hpp

@@ -1,227 +0,0 @@
-#ifndef CPPJIEBA_UNICODE_H
-#define CPPJIEBA_UNICODE_H
-
-#include <stdint.h>
-#include <stdlib.h>
-#include <string>
-#include <vector>
-#include <ostream>
-#include "limonp/LocalVector.hpp"
-
-namespace cppjieba {
-
-using std::string;
-using std::vector;
-
-typedef uint32_t Rune;
-
-struct Word {
-  string word;
-  uint32_t offset;
-  uint32_t unicode_offset;
-  uint32_t unicode_length;
-  Word(const string& w, uint32_t o)
-   : word(w), offset(o) {
-  }
-  Word(const string& w, uint32_t o, uint32_t unicode_offset, uint32_t unicode_length)
-          : word(w), offset(o), unicode_offset(unicode_offset), unicode_length(unicode_length) {
-  }
-}; // struct Word
-
-inline std::ostream& operator << (std::ostream& os, const Word& w) {
-  return os << "{\"word\": \"" << w.word << "\", \"offset\": " << w.offset << "}";
-}
-
-struct RuneStr {
-  Rune rune;
-  uint32_t offset;
-  uint32_t len;
-  uint32_t unicode_offset;
-  uint32_t unicode_length;
-  RuneStr(): rune(0), offset(0), len(0) {
-  }
-  RuneStr(Rune r, uint32_t o, uint32_t l)
-    : rune(r), offset(o), len(l) {
-  }
-  RuneStr(Rune r, uint32_t o, uint32_t l, uint32_t unicode_offset, uint32_t unicode_length)
-          : rune(r), offset(o), len(l), unicode_offset(unicode_offset), unicode_length(unicode_length) {
-  }
-}; // struct RuneStr
-
-inline std::ostream& operator << (std::ostream& os, const RuneStr& r) {
-  return os << "{\"rune\": \"" << r.rune << "\", \"offset\": " << r.offset << ", \"len\": " << r.len << "}";
-}
-
-typedef limonp::LocalVector<Rune> Unicode;
-typedef limonp::LocalVector<struct RuneStr> RuneStrArray;
-
-// [left, right]
-struct WordRange {
-  RuneStrArray::const_iterator left;
-  RuneStrArray::const_iterator right;
-  WordRange(RuneStrArray::const_iterator l, RuneStrArray::const_iterator r)
-   : left(l), right(r) {
-  }
-  size_t Length() const {
-    return right - left + 1;
-  }
-  bool IsAllAscii() const {
-    for (RuneStrArray::const_iterator iter = left; iter <= right; ++iter) {
-      if (iter->rune >= 0x80) {
-        return false;
-      }
-    }
-    return true;
-  }
-}; // struct WordRange
-
-struct RuneStrLite {
-  uint32_t rune;
-  uint32_t len;
-  RuneStrLite(): rune(0), len(0) {
-  }
-  RuneStrLite(uint32_t r, uint32_t l): rune(r), len(l) {
-  }
-}; // struct RuneStrLite
-
-inline RuneStrLite DecodeRuneInString(const char* str, size_t len) {
-  RuneStrLite rp(0, 0);
-  if (str == NULL || len == 0) {
-    return rp;
-  }
-  if (!(str[0] & 0x80)) { // 0xxxxxxx
-    // 7bit, total 7bit
-    rp.rune = (uint8_t)(str[0]) & 0x7f;
-    rp.len = 1;
-  } else if ((uint8_t)str[0] <= 0xdf &&  1 < len) { 
-    // 110xxxxxx
-    // 5bit, total 5bit
-    rp.rune = (uint8_t)(str[0]) & 0x1f;
-
-    // 6bit, total 11bit
-    rp.rune <<= 6;
-    rp.rune |= (uint8_t)(str[1]) & 0x3f;
-    rp.len = 2;
-  } else if((uint8_t)str[0] <= 0xef && 2 < len) { // 1110xxxxxx
-    // 4bit, total 4bit
-    rp.rune = (uint8_t)(str[0]) & 0x0f;
-
-    // 6bit, total 10bit
-    rp.rune <<= 6;
-    rp.rune |= (uint8_t)(str[1]) & 0x3f;
-
-    // 6bit, total 16bit
-    rp.rune <<= 6;
-    rp.rune |= (uint8_t)(str[2]) & 0x3f;
-
-    rp.len = 3;
-  } else if((uint8_t)str[0] <= 0xf7 && 3 < len) { // 11110xxxx
-    // 3bit, total 3bit
-    rp.rune = (uint8_t)(str[0]) & 0x07;
-
-    // 6bit, total 9bit
-    rp.rune <<= 6;
-    rp.rune |= (uint8_t)(str[1]) & 0x3f;
-
-    // 6bit, total 15bit
-    rp.rune <<= 6;
-    rp.rune |= (uint8_t)(str[2]) & 0x3f;
-
-    // 6bit, total 21bit
-    rp.rune <<= 6;
-    rp.rune |= (uint8_t)(str[3]) & 0x3f;
-
-    rp.len = 4;
-  } else {
-    rp.rune = 0;
-    rp.len = 0;
-  }
-  return rp;
-}
-
-inline bool DecodeRunesInString(const char* s, size_t len, RuneStrArray& runes) {
-  runes.clear();
-  runes.reserve(len / 2);
-  for (uint32_t i = 0, j = 0; i < len;) {
-    RuneStrLite rp = DecodeRuneInString(s + i, len - i);
-    if (rp.len == 0) {
-      runes.clear();
-      return false;
-    }
-    RuneStr x(rp.rune, i, rp.len, j, 1);
-    runes.push_back(x);
-    i += rp.len;
-    ++j;
-  }
-  return true;
-}
-
-inline bool DecodeRunesInString(const string& s, RuneStrArray& runes) {
-  return DecodeRunesInString(s.c_str(), s.size(), runes);
-}
-
-inline bool DecodeRunesInString(const char* s, size_t len, Unicode& unicode) {
-  unicode.clear();
-  RuneStrArray runes;
-  if (!DecodeRunesInString(s, len, runes)) {
-    return false;
-  }
-  unicode.reserve(runes.size());
-  for (size_t i = 0; i < runes.size(); i++) {
-    unicode.push_back(runes[i].rune);
-  }
-  return true;
-}
-
-inline bool IsSingleWord(const string& str) {
-  RuneStrLite rp = DecodeRuneInString(str.c_str(), str.size());
-  return rp.len == str.size();
-}
-
-inline bool DecodeRunesInString(const string& s, Unicode& unicode) {
-  return DecodeRunesInString(s.c_str(), s.size(), unicode);
-}
-
-inline Unicode DecodeRunesInString(const string& s) {
-  Unicode result;
-  DecodeRunesInString(s, result);
-  return result;
-}
-
-
-// [left, right]
-inline Word GetWordFromRunes(const string& s, RuneStrArray::const_iterator left, RuneStrArray::const_iterator right) {
-  assert(right->offset >= left->offset);
-  uint32_t len = right->offset - left->offset + right->len;
-  uint32_t unicode_length = right->unicode_offset - left->unicode_offset + right->unicode_length;
-  return Word(s.substr(left->offset, len), left->offset, left->unicode_offset, unicode_length);
-}
-
-inline string GetStringFromRunes(const string& s, RuneStrArray::const_iterator left, RuneStrArray::const_iterator right) {
-  assert(right->offset >= left->offset);
-  uint32_t len = right->offset - left->offset + right->len;
-  return s.substr(left->offset, len);
-}
-
-inline void GetWordsFromWordRanges(const string& s, const vector<WordRange>& wrs, vector<Word>& words) {
-  for (size_t i = 0; i < wrs.size(); i++) {
-    words.push_back(GetWordFromRunes(s, wrs[i].left, wrs[i].right));
-  }
-}
-
-inline vector<Word> GetWordsFromWordRanges(const string& s, const vector<WordRange>& wrs) {
-  vector<Word> result;
-  GetWordsFromWordRanges(s, wrs, result);
-  return result;
-}
-
-inline void GetStringsFromWords(const vector<Word>& words, vector<string>& strs) {
-  strs.resize(words.size());
-  for (size_t i = 0; i < words.size(); ++i) {
-    strs[i] = words[i].word;
-  }
-}
-
-} // namespace cppjieba
-
-#endif // CPPJIEBA_UNICODE_H