cppjieba_rb 0.3.3 → 0.4.0

data/ext/cppjieba/include/cppjieba/MPSegment.hpp

@@ -1,137 +0,0 @@
-#ifndef CPPJIEBA_MPSEGMENT_H
-#define CPPJIEBA_MPSEGMENT_H
-
-#include <algorithm>
-#include <set>
-#include <cassert>
-#include "limonp/Logging.hpp"
-#include "DictTrie.hpp"
-#include "SegmentTagged.hpp"
-#include "PosTagger.hpp"
-
-namespace cppjieba {
-
-class MPSegment: public SegmentTagged {
- public:
-  MPSegment(const string& dictPath, const string& userDictPath = "")
-    : dictTrie_(new DictTrie(dictPath, userDictPath)), isNeedDestroy_(true) {
-  }
-  MPSegment(const DictTrie* dictTrie)
-    : dictTrie_(dictTrie), isNeedDestroy_(false) {
-    assert(dictTrie_);
-  }
-  ~MPSegment() {
-    if (isNeedDestroy_) {
-      delete dictTrie_;
-    }
-  }
-
-  void Cut(const string& sentence, vector<string>& words) const {
-    Cut(sentence, words, MAX_WORD_LENGTH);
-  }
-
-  void Cut(const string& sentence,
-        vector<string>& words,
-        size_t max_word_len) const {
-    vector<Word> tmp;
-    Cut(sentence, tmp, max_word_len);
-    GetStringsFromWords(tmp, words);
-  }
-  void Cut(const string& sentence, 
-        vector<Word>& words, 
-        size_t max_word_len = MAX_WORD_LENGTH) const {
-    PreFilter pre_filter(symbols_, sentence);
-    PreFilter::Range range;
-    vector<WordRange> wrs;
-    wrs.reserve(sentence.size()/2);
-    while (pre_filter.HasNext()) {
-      range = pre_filter.Next();
-      Cut(range.begin, range.end, wrs, max_word_len);
-    }
-    words.clear();
-    words.reserve(wrs.size());
-    GetWordsFromWordRanges(sentence, wrs, words);
-  }
-  void Cut(RuneStrArray::const_iterator begin,
-           RuneStrArray::const_iterator end,
-           vector<WordRange>& words,
-           size_t max_word_len = MAX_WORD_LENGTH) const {
-    vector<Dag> dags;
-    dictTrie_->Find(begin, 
-          end, 
-          dags,
-          max_word_len);
-    CalcDP(dags);
-    CutByDag(begin, end, dags, words);
-  }
-
-  const DictTrie* GetDictTrie() const {
-    return dictTrie_;
-  }
-
-  bool Tag(const string& src, vector<pair<string, string> >& res) const {
-    return tagger_.Tag(src, res, *this);
-  }
-
-  bool IsUserDictSingleChineseWord(const Rune& value) const {
-    return dictTrie_->IsUserDictSingleChineseWord(value);
-  }
- private:
-  void CalcDP(vector<Dag>& dags) const {
-    size_t nextPos;
-    const DictUnit* p;
-    double val;
-
-    for (vector<Dag>::reverse_iterator rit = dags.rbegin(); rit != dags.rend(); rit++) {
-      rit->pInfo = NULL;
-      rit->weight = MIN_DOUBLE;
-      assert(!rit->nexts.empty());
-      for (LocalVector<pair<size_t, const DictUnit*> >::const_iterator it = rit->nexts.begin(); it != rit->nexts.end(); it++) {
-        nextPos = it->first;
-        p = it->second;
-        val = 0.0;
-        if (nextPos + 1 < dags.size()) {
-          val += dags[nextPos + 1].weight;
-        }
-
-        if (p) {
-          val += p->weight;
-        } else {
-          val += dictTrie_->GetMinWeight();
-        }
-        if (val > rit->weight) {
-          rit->pInfo = p;
-          rit->weight = val;
-        }
-      }
-    }
-  }
-  void CutByDag(RuneStrArray::const_iterator begin, 
-        RuneStrArray::const_iterator end, 
-        const vector<Dag>& dags, 
-        vector<WordRange>& words) const {
-    size_t i = 0;
-    while (i < dags.size()) {
-      const DictUnit* p = dags[i].pInfo;
-      if (p) {
-        assert(p->word.size() >= 1);
-        WordRange wr(begin + i, begin + i + p->word.size() - 1);
-        words.push_back(wr);
-        i += p->word.size();
-      } else { //single chinese word
-        WordRange wr(begin + i, begin + i);
-        words.push_back(wr);
-        i++;
-      }
-    }
-  }
-
-  const DictTrie* dictTrie_;
-  bool isNeedDestroy_;
-  PosTagger tagger_;
-
-}; // class MPSegment
-
-} // namespace cppjieba
-
-#endif

data/ext/cppjieba/include/cppjieba/MixSegment.hpp

@@ -1,109 +0,0 @@
-#ifndef CPPJIEBA_MIXSEGMENT_H
-#define CPPJIEBA_MIXSEGMENT_H
-
-#include <cassert>
-#include "MPSegment.hpp"
-#include "HMMSegment.hpp"
-#include "limonp/StringUtil.hpp"
-#include "PosTagger.hpp"
-
-namespace cppjieba {
-class MixSegment: public SegmentTagged {
- public:
-  MixSegment(const string& mpSegDict, const string& hmmSegDict, 
-        const string& userDict = "") 
-    : mpSeg_(mpSegDict, userDict), 
-      hmmSeg_(hmmSegDict) {
-  }
-  MixSegment(const DictTrie* dictTrie, const HMMModel* model) 
-    : mpSeg_(dictTrie), hmmSeg_(model) {
-  }
-  ~MixSegment() {
-  }
-
-  void Cut(const string& sentence, vector<string>& words) const {
-    Cut(sentence, words, true);
-  }
-  void Cut(const string& sentence, vector<string>& words, bool hmm) const {
-    vector<Word> tmp;
-    Cut(sentence, tmp, hmm);
-    GetStringsFromWords(tmp, words);
-  }
-  void Cut(const string& sentence, vector<Word>& words, bool hmm = true) const {
-    PreFilter pre_filter(symbols_, sentence);
-    PreFilter::Range range;
-    vector<WordRange> wrs;
-    wrs.reserve(sentence.size() / 2);
-    while (pre_filter.HasNext()) {
-      range = pre_filter.Next();
-      Cut(range.begin, range.end, wrs, hmm);
-    }
-    words.clear();
-    words.reserve(wrs.size());
-    GetWordsFromWordRanges(sentence, wrs, words);
-  }
-
-  void Cut(RuneStrArray::const_iterator begin, RuneStrArray::const_iterator end, vector<WordRange>& res, bool hmm) const {
-    if (!hmm) {
-      mpSeg_.Cut(begin, end, res);
-      return;
-    }
-    vector<WordRange> words;
-    assert(end >= begin);
-    words.reserve(end - begin);
-    mpSeg_.Cut(begin, end, words);
-
-    vector<WordRange> hmmRes;
-    hmmRes.reserve(end - begin);
-    for (size_t i = 0; i < words.size(); i++) {
-      //if mp Get a word, it's ok, put it into result
-      if (words[i].left != words[i].right || (words[i].left == words[i].right && mpSeg_.IsUserDictSingleChineseWord(words[i].left->rune))) {
-        res.push_back(words[i]);
-        continue;
-      }
-
-      // if mp Get a single one and it is not in userdict, collect it in sequence
-      size_t j = i;
-      while (j < words.size() && words[j].left == words[j].right && !mpSeg_.IsUserDictSingleChineseWord(words[j].left->rune)) {
-        j++;
-      }
-
-      // Cut the sequence with hmm
-      assert(j - 1 >= i);
-      // TODO
-      hmmSeg_.Cut(words[i].left, words[j - 1].left + 1, hmmRes);
-      //put hmm result to result
-      for (size_t k = 0; k < hmmRes.size(); k++) {
-        res.push_back(hmmRes[k]);
-      }
-
-      //clear tmp vars
-      hmmRes.clear();
-
-      //let i jump over this piece
-      i = j - 1;
-    }
-  }
-
-  const DictTrie* GetDictTrie() const {
-    return mpSeg_.GetDictTrie();
-  }
-
-  bool Tag(const string& src, vector<pair<string, string> >& res) const {
-    return tagger_.Tag(src, res, *this);
-  }
-
-  string LookupTag(const string &str) const {
-    return tagger_.LookupTag(str, *this);
-  }
-
- private:
-  MPSegment mpSeg_;
-  HMMSegment hmmSeg_;
-  PosTagger tagger_;
-
-}; // class MixSegment
-
-} // namespace cppjieba
-
-#endif

data/ext/cppjieba/include/cppjieba/PosTagger.hpp

@@ -1,77 +0,0 @@
-#ifndef CPPJIEBA_POS_TAGGING_H
-#define CPPJIEBA_POS_TAGGING_H
-
-#include "limonp/StringUtil.hpp"
-#include "SegmentTagged.hpp"
-#include "DictTrie.hpp"
-
-namespace cppjieba {
-using namespace limonp;
-
-static const char* const POS_M = "m";
-static const char* const POS_ENG = "eng";
-static const char* const POS_X = "x";
-
-class PosTagger {
- public:
-  PosTagger() {
-  }
-  ~PosTagger() {
-  }
-
-  bool Tag(const string& src, vector<pair<string, string> >& res, const SegmentTagged& segment) const {
-    vector<string> CutRes;
-    segment.Cut(src, CutRes);
-
-    for (vector<string>::iterator itr = CutRes.begin(); itr != CutRes.end(); ++itr) {
-      res.push_back(make_pair(*itr, LookupTag(*itr, segment)));
-    }
-    return !res.empty();
-  }
-
-  string LookupTag(const string &str, const SegmentTagged& segment) const {
-    const DictUnit *tmp = NULL;
-    RuneStrArray runes;
-    const DictTrie * dict = segment.GetDictTrie();
-    assert(dict != NULL);
-      if (!DecodeRunesInString(str, runes)) {
-        XLOG(ERROR) << "Decode failed.";
-        return POS_X;
-      }
-      tmp = dict->Find(runes.begin(), runes.end());
-      if (tmp == NULL || tmp->tag.empty()) {
-        return SpecialRule(runes);
-      } else {
-        return tmp->tag;
-      }
-  }
-
- private:
-  const char* SpecialRule(const RuneStrArray& unicode) const {
-    size_t m = 0;
-    size_t eng = 0;
-    for (size_t i = 0; i < unicode.size() && eng < unicode.size() / 2; i++) {
-      if (unicode[i].rune < 0x80) {
-        eng ++;
-        if ('0' <= unicode[i].rune && unicode[i].rune <= '9') {
-          m++;
-        }
-      }
-    }
-    // ascii char is not found
-    if (eng == 0) {
-      return POS_X;
-    }
-    // all the ascii is number char
-    if (m == eng) {
-      return POS_M;
-    }
-    // the ascii chars contain english letter
-    return POS_ENG;
-  }
-
-}; // class PosTagger
-
-} // namespace cppjieba
-
-#endif

data/ext/cppjieba/include/cppjieba/PreFilter.hpp

@@ -1,54 +0,0 @@
-#ifndef CPPJIEBA_PRE_FILTER_H
-#define CPPJIEBA_PRE_FILTER_H
-
-#include "Trie.hpp"
-#include "limonp/Logging.hpp"
-
-namespace cppjieba {
-
-class PreFilter {
- public:
-  //TODO use WordRange instead of Range
-  struct Range {
-    RuneStrArray::const_iterator begin;
-    RuneStrArray::const_iterator end;
-  }; // struct Range
-
-  PreFilter(const unordered_set<Rune>& symbols, 
-        const string& sentence)
-    : symbols_(symbols) {
-    if (!DecodeRunesInString(sentence, sentence_)) {
-      XLOG(ERROR) << "decode failed. "; 
-    }
-    cursor_ = sentence_.begin();
-  }
-  ~PreFilter() {
-  }
-  bool HasNext() const {
-    return cursor_ != sentence_.end();
-  }
-  Range Next() {
-    Range range;
-    range.begin = cursor_;
-    while (cursor_ != sentence_.end()) {
-      if (IsIn(symbols_, cursor_->rune)) {
-        if (range.begin == cursor_) {
-          cursor_ ++;
-        }
-        range.end = cursor_;
-        return range;
-      }
-      cursor_ ++;
-    }
-    range.end = sentence_.end();
-    return range;
-  }
- private:
-  RuneStrArray::const_iterator cursor_;
-  RuneStrArray sentence_;
-  const unordered_set<Rune>& symbols_;
-}; // class PreFilter
-
-} // namespace cppjieba
-
-#endif // CPPJIEBA_PRE_FILTER_H

data/ext/cppjieba/include/cppjieba/QuerySegment.hpp

@@ -1,90 +0,0 @@
-#ifndef CPPJIEBA_QUERYSEGMENT_H
-#define CPPJIEBA_QUERYSEGMENT_H
-
-#include <algorithm>
-#include <set>
-#include <cassert>
-#include "limonp/Logging.hpp"
-#include "DictTrie.hpp"
-#include "SegmentBase.hpp"
-#include "FullSegment.hpp"
-#include "MixSegment.hpp"
-#include "Unicode.hpp"
-#include "DictTrie.hpp"
-
-namespace cppjieba {
-class QuerySegment: public SegmentBase {
- public:
-  QuerySegment(const string& dict, const string& model, const string& userDict = "")
-    : mixSeg_(dict, model, userDict),
-      trie_(mixSeg_.GetDictTrie()) {
-  }
-  QuerySegment(const DictTrie* dictTrie, const HMMModel* model)
-    : mixSeg_(dictTrie, model), trie_(dictTrie) {
-  }
-  ~QuerySegment() {
-  }
-
-  void Cut(const string& sentence, vector<string>& words) const {
-    Cut(sentence, words, true);
-  }
-  void Cut(const string& sentence, vector<string>& words, bool hmm) const {
-    vector<Word> tmp;
-    Cut(sentence, tmp, hmm);
-    GetStringsFromWords(tmp, words);
-  }
-  void Cut(const string& sentence, vector<Word>& words, bool hmm = true) const {
-    PreFilter pre_filter(symbols_, sentence);
-    PreFilter::Range range;
-    vector<WordRange> wrs;
-    wrs.reserve(sentence.size()/2);
-    while (pre_filter.HasNext()) {
-      range = pre_filter.Next();
-      Cut(range.begin, range.end, wrs, hmm);
-    }
-    words.clear();
-    words.reserve(wrs.size());
-    GetWordsFromWordRanges(sentence, wrs, words);
-  }
-  void Cut(RuneStrArray::const_iterator begin, RuneStrArray::const_iterator end, vector<WordRange>& res, bool hmm) const {
-    //use mix Cut first
-    vector<WordRange> mixRes;
-    mixSeg_.Cut(begin, end, mixRes, hmm);
-
-    vector<WordRange> fullRes;
-    for (vector<WordRange>::const_iterator mixResItr = mixRes.begin(); mixResItr != mixRes.end(); mixResItr++) {
-      if (mixResItr->Length() > 2) {
-        for (size_t i = 0; i + 1 < mixResItr->Length(); i++) {
-          WordRange wr(mixResItr->left + i, mixResItr->left + i + 1);
-          if (trie_->Find(wr.left, wr.right + 1) != NULL) {
-            res.push_back(wr);
-          }
-        }
-      }
-      if (mixResItr->Length() > 3) {
-        for (size_t i = 0; i + 2 < mixResItr->Length(); i++) {
-          WordRange wr(mixResItr->left + i, mixResItr->left + i + 2);
-          if (trie_->Find(wr.left, wr.right + 1) != NULL) {
-            res.push_back(wr);
-          }
-        }
-      }
-      res.push_back(*mixResItr);
-    }
-  }
- private:
-  bool IsAllAscii(const Unicode& s) const {
-   for(size_t i = 0; i < s.size(); i++) {
-     if (s[i] >= 0x80) {
-       return false;
-     }
-   }
-   return true;
-  }
-  MixSegment mixSeg_;
-  const DictTrie* trie_;
-}; // QuerySegment
-
-} // namespace cppjieba
-
-#endif